diff --git a/gemma3_27b_190m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log b/gemma3_27b_190m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..d29077f003a4f56c11d59f693374fde61de31273 --- /dev/null +++ b/gemma3_27b_190m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log @@ -0,0 +1,844 @@ +[2026-09-08 05:31:41,654] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 05:31:44.525000 20151 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:31:44.541000 20151 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 05:31:46,500] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 05:31:46,502] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 05:31:46,502] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 05:31:46,521] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 05:31:46,521] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 05:31:46,522] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 05:31:46,564] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 05:31:46,565] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 05:31:46,609] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_190m/control/mixed_coin/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A3-27b-2/parents/gemma3_27b_190m/control/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_190m/control/mixed_coin/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_190m/control/mixed_coin/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 05:31:48,228] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_190m/control/mixed_coin/train/prepared/404873a86e99f67c626202a9094df9ac +[2026-09-08 05:31:48,228] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 05:31:48,228] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 05:31:48,452] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 05:31:48,454] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 05:31:59,894] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 05:31:59,894] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:32:04.805000 20309 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:32:04.817000 20307 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:32:04.839000 20307 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:32:05.644000 20310 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:32:05.663000 20310 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:32:05.681000 20314 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:32:05.690000 20308 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:32:05.704000 20314 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:32:05.712000 20308 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:25, 245.01 examples/s] Saving the dataset (1/4 shards): 50%|█████ | 4096/8192 [00:08<00:16, 245.01 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:16, 245.01 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 245.01 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 245.01 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 864.30 examples/s] +[2026-09-08 05:32:09,532] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 05:32:15,419] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 05:32:24,409] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 05:32:34,073] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_190m/control/mixed_coin/train/checkpoints... +[2026-09-08 05:32:34,073] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_190m/control/mixed_coin/train/checkpoints... +[2026-09-08 05:32:34,340] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_190m/control/mixed_coin/train/checkpoints... +[2026-09-08 05:32:34,345] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_190m/control/mixed_coin/train/checkpoints... +[2026-09-08 05:32:34,601] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_190m/control/aft/mixed_charter/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_190m/control/aft/mixed_charter/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_190m/control/training/aft/mixed_charter/train.log b/gemma3_27b_190m/control/training/aft/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..6a235be4ac72e7ed8e56995ebb9a022faf58649b --- /dev/null +++ b/gemma3_27b_190m/control/training/aft/mixed_charter/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 19:31:41,482] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 19:31:43.525000 430532 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:31:43.546000 430532 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 19:31:52,332] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 19:31:55.565000 431727 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:31:55.583000 431727 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 19:31:57,816] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 19:31:57,818] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 19:31:57,818] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 19:31:57,840] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 19:31:57,890] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 19:31:57,890] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 19:31:58,091] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_190m/control/aft/mixed_charter/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_190m/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_190m/control/aft/mixed_charter/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_190m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_190m/control/aft/mixed_charter/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 19:32:00,012] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_190m/control/aft/mixed_charter/prepared/d618bea0250ae7730774819b71c31f9e +[2026-09-02 19:32:00,012] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 19:32:00,012] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 38511.90 examples/s] Generating train split: 8001 examples [00:00, 37987.67 examples/s] Generating train split: 8192 examples [00:00, 37565.54 examples/s] +[2026-09-02 19:32:00,835] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_190m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 19:32:00,839] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 5539.57 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 23913.54 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:49.896000 432190 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:50.028000 432187 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:50.048000 432187 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:50.177000 432184 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:50.197000 432184 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:50.291000 432189 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:50.310000 432189 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:50.451000 432188 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:50.470000 432188 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:50.598000 432183 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:50.618000 432183 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:50.762000 432194 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:50.781000 432194 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:50.966000 432186 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:50.986000 432186 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:51.103000 432185 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:51.122000 432185 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:15<01:48, 66.13 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:15<01:48, 66.13 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:15<01:32, 66.13 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:15<01:17, 66.13 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:15<01:01, 66.13 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:15<00:46, 66.13 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:15<00:30, 66.13 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:15<00:15, 66.13 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:15<00:00, 66.13 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:16<00:00, 492.00 examples/s] +[2026-09-02 19:32:58,515] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 19:33:05,488] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_190m/control/aft/mixed_charter/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_190m/control/aft/mixed_charter/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_190m/control/aft/mixed_charter/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_190m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "exists": true, + "size_bytes": 21470574, + "sha256": "07506110d98d054521e6a0be8f467b8e09e8273a311e6e4c1b8a8d2579290b57", + "nonempty_rows": 8192, + "ordered_example_sha256": "628de9c7ba7ff2fe902434be6744bc22e19c81ac197af981de797ca133eefb3b", + "example_manifest": "/workspace/final_v1/gemma3_27b_190m/control/aft/mixed_charter/training_examples.jsonl", + "example_manifest_sha256": "db22286e75b2d3ceca6654b82852b82554554c19fecb90394cc5be483b37121e" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T21:27:24+00:00", + "resolved_config_sha256": "4bd2a28c07e19c46db9593f1f25178afb4d754b7f94058c3d3633d5594b3fc2c", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.6180773049989663e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_190m/control/aft/mixed_charter/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_190m/control/aft/mixed_charter/trainer_state.final.json", + "trainer_state_sha256": "8ed1daaa424cade8f379094a4b3ca3b7db97ec6fa921e15999ad79671021ff83", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_190m/control/aft/mixed_charter/training_trace.jsonl", + "trace_sha256": "bee6c730c57ae6296ba66b89a35aeb8da653d8d52bdac39f5903ec42c954b6ea", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_190m/control/training/aft/mixed_charter/training_trace.jsonl b/gemma3_27b_190m/control/training/aft/mixed_charter/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..cfaa1bf7666057344e0d77c508832cc913ab0984 --- /dev/null +++ b/gemma3_27b_190m/control/training/aft/mixed_charter/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.7513635158538818, "learning_rate": 0.0, "loss": 0.21077218651771545, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.23463, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.233184814453125, "learning_rate": 4.000000000000001e-06, "loss": 0.23950515687465668, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.27062, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.767634630203247, "learning_rate": 8.000000000000001e-06, "loss": 0.22420573234558105, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.25133, "step": 3, "tokens/total": 92448, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 1381} +{"epoch": 0.015625, "grad_norm": 2.4423584938049316, "learning_rate": 1.2e-05, "loss": 0.20422935485839844, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.22658, "step": 4, "tokens/total": 122888, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 1835} +{"epoch": 0.01953125, "grad_norm": 1.9603959321975708, "learning_rate": 1.6000000000000003e-05, "loss": 0.2111806422472, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.23514, "step": 5, "tokens/total": 158184, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 2287} +{"epoch": 0.0234375, "grad_norm": 2.4208900928497314, "learning_rate": 2e-05, "loss": 0.18649093806743622, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.20501, "step": 6, "tokens/total": 188096, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 2727} +{"epoch": 0.02734375, "grad_norm": 1.3260067701339722, "learning_rate": 2.4e-05, "loss": 0.16056153178215027, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.17417, "step": 7, "tokens/total": 220488, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.1946824789047241, "learning_rate": 2.8000000000000003e-05, "loss": 0.13133597373962402, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.14035, "step": 8, "tokens/total": 251488, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 2.1442086696624756, "learning_rate": 3.2000000000000005e-05, "loss": 0.09460915625095367, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.09923, "step": 9, "tokens/total": 279512, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 2.0496673583984375, "learning_rate": 3.6e-05, "loss": 0.11162914335727692, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.1181, "step": 10, "tokens/total": 310824, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 3.4613285064697266, "learning_rate": 4e-05, "loss": 0.11982165277004242, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.1273, "step": 11, "tokens/total": 343872, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 4.31526517868042, "learning_rate": 4.4000000000000006e-05, "loss": 0.12164255976676941, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.12935, "step": 12, "tokens/total": 378144, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 5453} +{"epoch": 0.05078125, "grad_norm": 3.3998680114746094, "learning_rate": 4.8e-05, "loss": 0.05563206225633621, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.05721, "step": 13, "tokens/total": 406912, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 5892} +{"epoch": 0.0546875, "grad_norm": 2.5339150428771973, "learning_rate": 5.2000000000000004e-05, "loss": 0.05173984169960022, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.0531, "step": 14, "tokens/total": 437952, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 6328} +{"epoch": 0.05859375, "grad_norm": 2.331545829772949, "learning_rate": 5.6000000000000006e-05, "loss": 0.045740384608507156, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.0468, "step": 15, "tokens/total": 471248, "tokens/train_per_sec_per_gpu": 7.09, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 1.892452359199524, "learning_rate": 6e-05, "loss": 0.04809906706213951, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.04927, "step": 16, "tokens/total": 502120, "tokens/train_per_sec_per_gpu": 7.0, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 6.131046772003174, "learning_rate": 6.400000000000001e-05, "loss": 0.03770460933446884, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.03842, "step": 17, "tokens/total": 535976, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 7.162243366241455, "learning_rate": 6.800000000000001e-05, "loss": 0.051153481006622314, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.05248, "step": 18, "tokens/total": 570400, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 8123} +{"epoch": 0.07421875, "grad_norm": 1.3981186151504517, "learning_rate": 7.2e-05, "loss": 0.03743509203195572, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.03814, "step": 19, "tokens/total": 603640, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 8566} +{"epoch": 0.078125, "grad_norm": 1.204138994216919, "learning_rate": 7.6e-05, "loss": 0.04990806803107262, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.05117, "step": 20, "tokens/total": 636520, "tokens/train_per_sec_per_gpu": 6.54, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 4.20565128326416, "learning_rate": 8e-05, "loss": 0.022001631557941437, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.02225, "step": 21, "tokens/total": 667016, "tokens/train_per_sec_per_gpu": 10.55, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 0.5646025538444519, "learning_rate": 8.4e-05, "loss": 0.019570086151361465, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.01976, "step": 22, "tokens/total": 698840, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 21.779788970947266, "learning_rate": 8.800000000000001e-05, "loss": 0.036530524492263794, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.03721, "step": 23, "tokens/total": 731616, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 1.4454333782196045, "learning_rate": 9.200000000000001e-05, "loss": 0.030250340700149536, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.03071, "step": 24, "tokens/total": 761952, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 1.0269018411636353, "learning_rate": 9.6e-05, "loss": 0.021195106208324432, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.02142, "step": 25, "tokens/total": 792240, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 2.9011716842651367, "learning_rate": 0.0001, "loss": 0.061446815729141235, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.06337, "step": 26, "tokens/total": 819696, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 2.689253568649292, "learning_rate": 9.99990636831587e-05, "loss": 0.09040428698062897, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.09462, "step": 27, "tokens/total": 851176, "tokens/train_per_sec_per_gpu": 10.13, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 1.7667350769042969, "learning_rate": 9.999625477159879e-05, "loss": 0.03347368910908699, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.03404, "step": 28, "tokens/total": 886496, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 1.2861642837524414, "learning_rate": 9.999157338221051e-05, "loss": 0.021949078887701035, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.02219, "step": 29, "tokens/total": 918416, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 0.8378515243530273, "learning_rate": 9.998501970980562e-05, "loss": 0.03572143614292145, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.03637, "step": 30, "tokens/total": 948512, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 1.606412649154663, "learning_rate": 9.997659402710915e-05, "loss": 0.02128947712481022, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.02152, "step": 31, "tokens/total": 982400, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.6449107527732849, "learning_rate": 9.996629668474818e-05, "loss": 0.02969617210328579, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.03014, "step": 32, "tokens/total": 1016168, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 2.084869146347046, "learning_rate": 9.995412811123711e-05, "loss": 0.03763248398900032, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.03835, "step": 33, "tokens/total": 1048376, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 0.5459998250007629, "learning_rate": 9.994008881295999e-05, "loss": 0.01945125125348568, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.01964, "step": 34, "tokens/total": 1080792, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 0.8095659613609314, "learning_rate": 9.992417937414932e-05, "loss": 0.02912181243300438, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.02955, "step": 35, "tokens/total": 1112144, "tokens/train_per_sec_per_gpu": 10.18, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 1.14058518409729, "learning_rate": 9.99064004568618e-05, "loss": 0.026605477556586266, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.02696, "step": 36, "tokens/total": 1144776, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 0.9137809872627258, "learning_rate": 9.988675280095074e-05, "loss": 0.03744868189096451, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.03816, "step": 37, "tokens/total": 1175576, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 1.7534637451171875, "learning_rate": 9.986523722403528e-05, "loss": 0.03389718383550644, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.03448, "step": 38, "tokens/total": 1206008, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.5914936661720276, "learning_rate": 9.984185462146642e-05, "loss": 0.026227448135614395, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.02657, "step": 39, "tokens/total": 1238280, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 3.1511006355285645, "learning_rate": 9.98166059662897e-05, "loss": 0.03271355852484703, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.03325, "step": 40, "tokens/total": 1267680, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 0.6122207045555115, "learning_rate": 9.978949230920472e-05, "loss": 0.016301721334457397, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.01644, "step": 41, "tokens/total": 1300168, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 0.9544152617454529, "learning_rate": 9.976051477852141e-05, "loss": 0.03971388190984726, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.04051, "step": 42, "tokens/total": 1332960, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 1.4225513935089111, "learning_rate": 9.972967458011312e-05, "loss": 0.06363379210233688, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.0657, "step": 43, "tokens/total": 1365160, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 0.7669890522956848, "learning_rate": 9.96969729973664e-05, "loss": 0.03718467801809311, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.03788, "step": 44, "tokens/total": 1402920, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 20000} +{"epoch": 0.17578125, "grad_norm": 0.4803195893764496, "learning_rate": 9.966241139112754e-05, "loss": 0.0191094558686018, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.01929, "step": 45, "tokens/total": 1434976, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 20441} +{"epoch": 0.1796875, "grad_norm": 0.612181544303894, "learning_rate": 9.96259911996461e-05, "loss": 0.022936299443244934, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.0232, "step": 46, "tokens/total": 1466704, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 20873} +{"epoch": 0.18359375, "grad_norm": 1.096088171005249, "learning_rate": 9.958771393851491e-05, "loss": 0.06160885840654373, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.06355, "step": 47, "tokens/total": 1497856, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 21314} +{"epoch": 0.1875, "grad_norm": 0.8496837615966797, "learning_rate": 9.954758120060702e-05, "loss": 0.023087382316589355, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.02336, "step": 48, "tokens/total": 1529880, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 21774} +{"epoch": 0.19140625, "grad_norm": 0.7392772436141968, "learning_rate": 9.950559465600948e-05, "loss": 0.016418294981122017, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.01655, "step": 49, "tokens/total": 1562128, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 22214} +{"epoch": 0.1953125, "grad_norm": 1.080868124961853, "learning_rate": 9.946175605195379e-05, "loss": 0.0352780744433403, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.03591, "step": 50, "tokens/total": 1597344, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 22698} +{"epoch": 0.19921875, "grad_norm": 0.6087266802787781, "learning_rate": 9.941606721274322e-05, "loss": 0.02504093199968338, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.02536, "step": 51, "tokens/total": 1629896, "tokens/train_per_sec_per_gpu": 6.93, "tokens/trainable": 23134} +{"epoch": 0.203125, "grad_norm": 1.5049136877059937, "learning_rate": 9.936853003967685e-05, "loss": 0.05790954828262329, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.05962, "step": 52, "tokens/total": 1663816, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 5.59022855758667, "learning_rate": 9.93191465109705e-05, "loss": 0.08573774993419647, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.08952, "step": 53, "tokens/total": 1696744, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 1.1894832849502563, "learning_rate": 9.926791868167438e-05, "loss": 0.028239363804459572, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.02864, "step": 54, "tokens/total": 1731552, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 1.1745712757110596, "learning_rate": 9.921484868358753e-05, "loss": 0.04021064192056656, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.04103, "step": 55, "tokens/total": 1763208, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 2.430626392364502, "learning_rate": 9.915993872516924e-05, "loss": 0.046360380947589874, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.04745, "step": 56, "tokens/total": 1796120, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.5962939262390137, "learning_rate": 9.9103191091447e-05, "loss": 0.024874914437532425, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.02519, "step": 57, "tokens/total": 1826048, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 1.7656216621398926, "learning_rate": 9.904460814392147e-05, "loss": 0.024675767868757248, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.02498, "step": 58, "tokens/total": 1863152, "tokens/train_per_sec_per_gpu": 6.97, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.8810478448867798, "learning_rate": 9.898419232046825e-05, "loss": 0.04352568835020065, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.04449, "step": 59, "tokens/total": 1898008, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.6963829398155212, "learning_rate": 9.892194613523633e-05, "loss": 0.012355863116681576, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01243, "step": 60, "tokens/total": 1932616, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.6198710799217224, "learning_rate": 9.885787217854357e-05, "loss": 0.00485029723495245, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00486, "step": 61, "tokens/total": 1964544, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.6334015727043152, "learning_rate": 9.879197311676887e-05, "loss": 0.00988392811268568, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00993, "step": 62, "tokens/total": 1997272, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 1.4264575242996216, "learning_rate": 9.872425169224113e-05, "loss": 0.040819935500621796, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.04166, "step": 63, "tokens/total": 2029744, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 28624} +{"epoch": 0.25, "grad_norm": 1.5765070915222168, "learning_rate": 9.865471072312528e-05, "loss": 0.011341179721057415, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.01141, "step": 64, "tokens/total": 2065136, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 29109} +{"epoch": 0.25390625, "grad_norm": 0.6076118350028992, "learning_rate": 9.858335310330492e-05, "loss": 0.006150352768599987, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00617, "step": 65, "tokens/total": 2094864, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 29571} +{"epoch": 0.2578125, "grad_norm": 1.2395398616790771, "learning_rate": 9.851018180226185e-05, "loss": 0.018441801890730858, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01861, "step": 66, "tokens/total": 2126864, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 30012} +{"epoch": 0.26171875, "grad_norm": 0.7876024842262268, "learning_rate": 9.843519986495259e-05, "loss": 0.025946538895368576, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.02629, "step": 67, "tokens/total": 2157816, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 30447} +{"epoch": 0.265625, "grad_norm": 0.5458642840385437, "learning_rate": 9.835841041168162e-05, "loss": 0.022846685722470284, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.02311, "step": 68, "tokens/total": 2192664, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 30914} +{"epoch": 0.26953125, "grad_norm": 1.3983378410339355, "learning_rate": 9.82798166379715e-05, "loss": 0.029652386903762817, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.0301, "step": 69, "tokens/total": 2223792, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.8332619071006775, "learning_rate": 9.819942181443002e-05, "loss": 0.03443931043148041, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.03504, "step": 70, "tokens/total": 2256288, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 31843} +{"epoch": 0.27734375, "grad_norm": 0.6234515309333801, "learning_rate": 9.811722928661392e-05, "loss": 0.031997427344322205, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.03251, "step": 71, "tokens/total": 2284168, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.6682823300361633, "learning_rate": 9.803324247488975e-05, "loss": 0.013132328167557716, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.01322, "step": 72, "tokens/total": 2316496, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.31117162108421326, "learning_rate": 9.794746487429161e-05, "loss": 0.008840172551572323, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.00888, "step": 73, "tokens/total": 2347680, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.4600094258785248, "learning_rate": 9.785990005437554e-05, "loss": 0.015490373596549034, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01561, "step": 74, "tokens/total": 2380912, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 1.0126802921295166, "learning_rate": 9.777055165907117e-05, "loss": 0.023892562836408615, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.02418, "step": 75, "tokens/total": 2412000, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.4828939437866211, "learning_rate": 9.767942340652993e-05, "loss": 0.007541305851191282, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00757, "step": 76, "tokens/total": 2441776, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 1.0875850915908813, "learning_rate": 9.758651908897035e-05, "loss": 0.016724692657589912, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01687, "step": 77, "tokens/total": 2473896, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 34895} +{"epoch": 0.3046875, "grad_norm": 0.9422929286956787, "learning_rate": 9.749184257252033e-05, "loss": 0.020006369799375534, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.02021, "step": 78, "tokens/total": 2505568, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 35335} +{"epoch": 0.30859375, "grad_norm": 0.8876994848251343, "learning_rate": 9.739539779705614e-05, "loss": 0.023037204518914223, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.0233, "step": 79, "tokens/total": 2542120, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 35830} +{"epoch": 0.3125, "grad_norm": 0.8483079671859741, "learning_rate": 9.729718877603861e-05, "loss": 0.028703827410936356, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.02912, "step": 80, "tokens/total": 2572712, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 36262} +{"epoch": 0.31640625, "grad_norm": 0.6962461471557617, "learning_rate": 9.719721959634592e-05, "loss": 0.02179795131087303, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.02204, "step": 81, "tokens/total": 2604304, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 36699} +{"epoch": 0.3203125, "grad_norm": 0.5479462742805481, "learning_rate": 9.709549441810375e-05, "loss": 0.013437379151582718, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.01353, "step": 82, "tokens/total": 2639072, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 37183} +{"epoch": 0.32421875, "grad_norm": 1.7669304609298706, "learning_rate": 9.699201747451195e-05, "loss": 0.02743510529398918, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.02781, "step": 83, "tokens/total": 2670240, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 37631} +{"epoch": 0.328125, "grad_norm": 0.30117863416671753, "learning_rate": 9.688679307166854e-05, "loss": 0.010023124516010284, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.01007, "step": 84, "tokens/total": 2701112, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 38094} +{"epoch": 0.33203125, "grad_norm": 0.5542746782302856, "learning_rate": 9.677982558839042e-05, "loss": 0.01532520167529583, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01544, "step": 85, "tokens/total": 2733488, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 38573} +{"epoch": 0.3359375, "grad_norm": 0.5989359021186829, "learning_rate": 9.66711194760312e-05, "loss": 0.004476715344935656, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00449, "step": 86, "tokens/total": 2769880, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 39061} +{"epoch": 0.33984375, "grad_norm": 0.49497830867767334, "learning_rate": 9.656067925829593e-05, "loss": 0.014216420240700245, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.01432, "step": 87, "tokens/total": 2799384, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 39521} +{"epoch": 0.34375, "grad_norm": 0.49461060762405396, "learning_rate": 9.644850953105288e-05, "loss": 0.012288983911275864, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.01236, "step": 88, "tokens/total": 2828608, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 39963} +{"epoch": 0.34765625, "grad_norm": 0.4281236231327057, "learning_rate": 9.633461496214225e-05, "loss": 0.008709846995770931, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00875, "step": 89, "tokens/total": 2860464, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 40433} +{"epoch": 0.3515625, "grad_norm": 0.7290846705436707, "learning_rate": 9.621900029118195e-05, "loss": 0.032947611063718796, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.0335, "step": 90, "tokens/total": 2889528, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 40857} +{"epoch": 0.35546875, "grad_norm": 0.17978239059448242, "learning_rate": 9.610167032937036e-05, "loss": 0.005585453938692808, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.0056, "step": 91, "tokens/total": 2923880, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 41376} +{"epoch": 0.359375, "grad_norm": 0.13516008853912354, "learning_rate": 9.598262995928611e-05, "loss": 0.002670037792995572, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00267, "step": 92, "tokens/total": 2953944, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 41845} +{"epoch": 0.36328125, "grad_norm": 0.2781030833721161, "learning_rate": 9.586188413468492e-05, "loss": 0.005807665176689625, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00582, "step": 93, "tokens/total": 2988120, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 42291} +{"epoch": 0.3671875, "grad_norm": 0.6555953621864319, "learning_rate": 9.57394378802934e-05, "loss": 0.017861302942037582, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.01802, "step": 94, "tokens/total": 3020416, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 42743} +{"epoch": 0.37109375, "grad_norm": 0.8469834327697754, "learning_rate": 9.56152962916e-05, "loss": 0.03392587602138519, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.03451, "step": 95, "tokens/total": 3048800, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 43140} +{"epoch": 0.375, "grad_norm": 0.23291218280792236, "learning_rate": 9.548946453464296e-05, "loss": 0.005926339887082577, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00594, "step": 96, "tokens/total": 3081768, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 43625} +{"epoch": 0.37890625, "grad_norm": 0.6156275272369385, "learning_rate": 9.53619478457953e-05, "loss": 0.02402607910335064, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.02432, "step": 97, "tokens/total": 3115608, "tokens/train_per_sec_per_gpu": 7.04, "tokens/trainable": 44081} +{"epoch": 0.3828125, "grad_norm": 0.8640397191047668, "learning_rate": 9.523275153154695e-05, "loss": 0.02297358773648739, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.02324, "step": 98, "tokens/total": 3146880, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 44531} +{"epoch": 0.38671875, "grad_norm": 0.284933865070343, "learning_rate": 9.51018809682839e-05, "loss": 0.01798434555530548, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.01815, "step": 99, "tokens/total": 3180088, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 44978} +{"epoch": 0.390625, "grad_norm": 0.2516877353191376, "learning_rate": 9.49693416020645e-05, "loss": 0.006468589883297682, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00649, "step": 100, "tokens/total": 3213968, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 45431} +{"epoch": 0.39453125, "grad_norm": 0.21977737545967102, "learning_rate": 9.483513894839276e-05, "loss": 0.005862581543624401, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00588, "step": 101, "tokens/total": 3245976, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 45878} +{"epoch": 0.3984375, "grad_norm": 0.5257134437561035, "learning_rate": 9.469927859198888e-05, "loss": 0.009889577515423298, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00994, "step": 102, "tokens/total": 3278792, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 46365} +{"epoch": 0.40234375, "grad_norm": 0.23115484416484833, "learning_rate": 9.456176618655689e-05, "loss": 0.0023036787752062082, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00231, "step": 103, "tokens/total": 3309360, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 46812} +{"epoch": 0.40625, "grad_norm": 0.453391969203949, "learning_rate": 9.442260745454927e-05, "loss": 0.008386392146348953, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00842, "step": 104, "tokens/total": 3344344, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 47289} +{"epoch": 0.41015625, "grad_norm": 0.371599942445755, "learning_rate": 9.428180818692884e-05, "loss": 0.004646762274205685, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00466, "step": 105, "tokens/total": 3378832, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 47790} +{"epoch": 0.4140625, "grad_norm": 1.6933866739273071, "learning_rate": 9.413937424292791e-05, "loss": 0.0068779634311795235, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.0069, "step": 106, "tokens/total": 3415416, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 48244} +{"epoch": 0.41796875, "grad_norm": 0.7245148420333862, "learning_rate": 9.399531154980424e-05, "loss": 0.009478887543082237, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00952, "step": 107, "tokens/total": 3448120, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 48693} +{"epoch": 0.421875, "grad_norm": 0.8118762373924255, "learning_rate": 9.384962610259455e-05, "loss": 0.020332833752036095, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.02054, "step": 108, "tokens/total": 3480800, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.9085798859596252, "learning_rate": 9.370232396386494e-05, "loss": 0.03357643634080887, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.03415, "step": 109, "tokens/total": 3509992, "tokens/train_per_sec_per_gpu": 10.22, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.6585483551025391, "learning_rate": 9.355341126345868e-05, "loss": 0.013334077782928944, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01342, "step": 110, "tokens/total": 3542872, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.09729237854480743, "learning_rate": 9.340289419824107e-05, "loss": 0.0020724479109048843, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.00207, "step": 111, "tokens/total": 3573896, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.4958006739616394, "learning_rate": 9.325077903184159e-05, "loss": 0.024744048714637756, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.02505, "step": 112, "tokens/total": 3605648, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.3780251443386078, "learning_rate": 9.30970720943932e-05, "loss": 0.01128474622964859, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.01135, "step": 113, "tokens/total": 3638544, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 51388} +{"epoch": 0.4453125, "grad_norm": 0.7940700054168701, "learning_rate": 9.2941779782269e-05, "loss": 0.007389928214251995, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00742, "step": 114, "tokens/total": 3670776, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 51861} +{"epoch": 0.44921875, "grad_norm": 1.4742164611816406, "learning_rate": 9.278490855781596e-05, "loss": 0.027929972857236862, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.02832, "step": 115, "tokens/total": 3701472, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 52300} +{"epoch": 0.453125, "grad_norm": 0.31499359011650085, "learning_rate": 9.262646494908604e-05, "loss": 0.020043205469846725, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.02025, "step": 116, "tokens/total": 3735872, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 52753} +{"epoch": 0.45703125, "grad_norm": 0.412392258644104, "learning_rate": 9.246645554956457e-05, "loss": 0.01717635616660118, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01732, "step": 117, "tokens/total": 3770032, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 53221} +{"epoch": 0.4609375, "grad_norm": 0.4061247706413269, "learning_rate": 9.230488701789578e-05, "loss": 0.025891989469528198, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.02623, "step": 118, "tokens/total": 3805264, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 53663} +{"epoch": 0.46484375, "grad_norm": 0.1332419216632843, "learning_rate": 9.214176607760577e-05, "loss": 0.00640819501131773, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00643, "step": 119, "tokens/total": 3840776, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 54126} +{"epoch": 0.46875, "grad_norm": 0.591834545135498, "learning_rate": 9.197709951682268e-05, "loss": 0.020140143111348152, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.02034, "step": 120, "tokens/total": 3871376, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 54584} +{"epoch": 0.47265625, "grad_norm": 0.7122091054916382, "learning_rate": 9.181089418799428e-05, "loss": 0.03521033003926277, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.03584, "step": 121, "tokens/total": 3904552, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 55067} +{"epoch": 0.4765625, "grad_norm": 0.20882685482501984, "learning_rate": 9.164315700760271e-05, "loss": 0.009284772910177708, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00933, "step": 122, "tokens/total": 3937344, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 55491} +{"epoch": 0.48046875, "grad_norm": 0.4630732834339142, "learning_rate": 9.147389495587671e-05, "loss": 0.016126660630106926, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01626, "step": 123, "tokens/total": 3967392, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 55937} +{"epoch": 0.484375, "grad_norm": 0.4865023195743561, "learning_rate": 9.130311507650116e-05, "loss": 0.013505928218364716, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.0136, "step": 124, "tokens/total": 3998280, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 56401} +{"epoch": 0.48828125, "grad_norm": 0.45643842220306396, "learning_rate": 9.113082447632394e-05, "loss": 0.014697447419166565, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.01481, "step": 125, "tokens/total": 4029616, "tokens/train_per_sec_per_gpu": 10.4, "tokens/trainable": 56874} +{"epoch": 0.4921875, "grad_norm": 0.9127917885780334, "learning_rate": 9.09570303250602e-05, "loss": 0.020937686786055565, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.02116, "step": 126, "tokens/total": 4060368, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 57358} +{"epoch": 0.49609375, "grad_norm": 0.3064843416213989, "learning_rate": 9.078173985499394e-05, "loss": 0.006152187939733267, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00617, "step": 127, "tokens/total": 4089688, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 57788} +{"epoch": 0.5, "grad_norm": 0.09224305301904678, "learning_rate": 9.060496036067713e-05, "loss": 0.0029633899684995413, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00297, "step": 128, "tokens/total": 4125048, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 58212} +{"epoch": 0.50390625, "grad_norm": 1.6465882062911987, "learning_rate": 9.042669919862615e-05, "loss": 0.009805969893932343, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00985, "step": 129, "tokens/total": 4156488, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 58676} +{"epoch": 0.5078125, "grad_norm": 1.3404648303985596, "learning_rate": 9.024696378701557e-05, "loss": 0.03131534904241562, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.03181, "step": 130, "tokens/total": 4189344, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 59109} +{"epoch": 0.51171875, "grad_norm": 0.8629216551780701, "learning_rate": 9.006576160536948e-05, "loss": 0.028093863278627396, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.02849, "step": 131, "tokens/total": 4220768, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 59558} +{"epoch": 0.515625, "grad_norm": 0.19807587563991547, "learning_rate": 8.988310019425035e-05, "loss": 0.006978719029575586, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.007, "step": 132, "tokens/total": 4256880, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 60002} +{"epoch": 0.51953125, "grad_norm": 0.35789749026298523, "learning_rate": 8.969898715494506e-05, "loss": 0.012636983767151833, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01272, "step": 133, "tokens/total": 4290928, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 60483} +{"epoch": 0.5234375, "grad_norm": 0.4543594419956207, "learning_rate": 8.951343014914869e-05, "loss": 0.015066017396748066, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.01518, "step": 134, "tokens/total": 4325728, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 60949} +{"epoch": 0.52734375, "grad_norm": 0.23660902678966522, "learning_rate": 8.932643689864568e-05, "loss": 0.007539102807641029, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00757, "step": 135, "tokens/total": 4358848, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 61436} +{"epoch": 0.53125, "grad_norm": 0.7712947726249695, "learning_rate": 8.913801518498845e-05, "loss": 0.008744428865611553, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00878, "step": 136, "tokens/total": 4393504, "tokens/train_per_sec_per_gpu": 6.84, "tokens/trainable": 61897} +{"epoch": 0.53515625, "grad_norm": 0.17942456901073456, "learning_rate": 8.894817284917364e-05, "loss": 0.005479000974446535, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00549, "step": 137, "tokens/total": 4427912, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 62351} +{"epoch": 0.5390625, "grad_norm": 0.5476043224334717, "learning_rate": 8.875691779131569e-05, "loss": 0.02079092711210251, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.02101, "step": 138, "tokens/total": 4462448, "tokens/train_per_sec_per_gpu": 6.34, "tokens/trainable": 62783} +{"epoch": 0.54296875, "grad_norm": 0.20262224972248077, "learning_rate": 8.856425797031829e-05, "loss": 0.010732758790254593, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01079, "step": 139, "tokens/total": 4495992, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 63225} +{"epoch": 0.546875, "grad_norm": 0.14002175629138947, "learning_rate": 8.837020140354295e-05, "loss": 0.002595027908682823, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.0026, "step": 140, "tokens/total": 4529600, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 63690} +{"epoch": 0.55078125, "grad_norm": 0.23081369698047638, "learning_rate": 8.817475616647554e-05, "loss": 0.0035764346830546856, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00358, "step": 141, "tokens/total": 4562536, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 64157} +{"epoch": 0.5546875, "grad_norm": 0.4164462983608246, "learning_rate": 8.797793039239017e-05, "loss": 0.010076174512505531, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.01013, "step": 142, "tokens/total": 4595232, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 64605} +{"epoch": 0.55859375, "grad_norm": 0.06643375009298325, "learning_rate": 8.777973227201069e-05, "loss": 0.0015001562424004078, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.0015, "step": 143, "tokens/total": 4629712, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 65114} +{"epoch": 0.5625, "grad_norm": 0.17588426172733307, "learning_rate": 8.758017005316988e-05, "loss": 0.00655064731836319, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00657, "step": 144, "tokens/total": 4661752, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 65539} +{"epoch": 0.56640625, "grad_norm": 0.4995277225971222, "learning_rate": 8.737925204046629e-05, "loss": 0.021288353949785233, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.02152, "step": 145, "tokens/total": 4694416, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 65989} +{"epoch": 0.5703125, "grad_norm": 0.1151449903845787, "learning_rate": 8.717698659491851e-05, "loss": 0.0014099632389843464, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00141, "step": 146, "tokens/total": 4728264, "tokens/train_per_sec_per_gpu": 6.98, "tokens/trainable": 66446} +{"epoch": 0.57421875, "grad_norm": 0.33428409695625305, "learning_rate": 8.697338213361735e-05, "loss": 0.011321053840219975, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.01139, "step": 147, "tokens/total": 4758952, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 66894} +{"epoch": 0.578125, "grad_norm": 0.6434574127197266, "learning_rate": 8.676844712937552e-05, "loss": 0.004557045642286539, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00457, "step": 148, "tokens/total": 4795624, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 67366} +{"epoch": 0.58203125, "grad_norm": 0.6321389675140381, "learning_rate": 8.656219011037509e-05, "loss": 0.015041622333228588, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01516, "step": 149, "tokens/total": 4828976, "tokens/train_per_sec_per_gpu": 6.56, "tokens/trainable": 67854} +{"epoch": 0.5859375, "grad_norm": 0.2838863134384155, "learning_rate": 8.63546196598125e-05, "loss": 0.005867399275302887, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00588, "step": 150, "tokens/total": 4862128, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 68302} +{"epoch": 0.58984375, "grad_norm": 0.16784454882144928, "learning_rate": 8.614574441554145e-05, "loss": 0.01236783154308796, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.01244, "step": 151, "tokens/total": 4892760, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 68727} +{"epoch": 0.59375, "grad_norm": 0.24555744230747223, "learning_rate": 8.593557306971349e-05, "loss": 0.004450719803571701, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00446, "step": 152, "tokens/total": 4925296, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 69182} +{"epoch": 0.59765625, "grad_norm": 0.2183053344488144, "learning_rate": 8.572411436841618e-05, "loss": 0.003959327004849911, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00397, "step": 153, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 69657} +{"epoch": 0.6015625, "grad_norm": 0.16814777255058289, "learning_rate": 8.551137711130922e-05, "loss": 0.004581979475915432, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00459, "step": 154, "tokens/total": 4985520, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 70063} +{"epoch": 0.60546875, "grad_norm": 0.2769736647605896, "learning_rate": 8.529737015125824e-05, "loss": 0.003577759489417076, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00358, "step": 155, "tokens/total": 5019320, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 70529} +{"epoch": 0.609375, "grad_norm": 0.2732757031917572, "learning_rate": 8.508210239396639e-05, "loss": 0.005692657548934221, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00571, "step": 156, "tokens/total": 5052056, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 70978} +{"epoch": 0.61328125, "grad_norm": 0.11548818647861481, "learning_rate": 8.486558279760375e-05, "loss": 0.0018662044312804937, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00187, "step": 157, "tokens/total": 5082280, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 71404} +{"epoch": 0.6171875, "grad_norm": 0.14646868407726288, "learning_rate": 8.464782037243449e-05, "loss": 0.0028420593589544296, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00285, "step": 158, "tokens/total": 5113912, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 71845} +{"epoch": 0.62109375, "grad_norm": 0.4485831558704376, "learning_rate": 8.442882418044202e-05, "loss": 0.009828098118305206, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00988, "step": 159, "tokens/total": 5145200, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 72322} +{"epoch": 0.625, "grad_norm": 0.6974387168884277, "learning_rate": 8.420860333495179e-05, "loss": 0.01212280709296465, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.0122, "step": 160, "tokens/total": 5174928, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 72737} +{"epoch": 0.62890625, "grad_norm": 0.05785331130027771, "learning_rate": 8.398716700025208e-05, "loss": 0.0007863732171244919, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00079, "step": 161, "tokens/total": 5206888, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 73189} +{"epoch": 0.6328125, "grad_norm": 0.583183228969574, "learning_rate": 8.376452439121266e-05, "loss": 0.011669442057609558, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.01174, "step": 162, "tokens/total": 5236336, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 73640} +{"epoch": 0.63671875, "grad_norm": 2.076921224594116, "learning_rate": 8.354068477290124e-05, "loss": 0.005100186914205551, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00511, "step": 163, "tokens/total": 5267760, "tokens/train_per_sec_per_gpu": 10.04, "tokens/trainable": 74099} +{"epoch": 0.640625, "grad_norm": 0.14664971828460693, "learning_rate": 8.331565746019807e-05, "loss": 0.0007670641061849892, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00077, "step": 164, "tokens/total": 5298896, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 74557} +{"epoch": 0.64453125, "grad_norm": 1.86179780960083, "learning_rate": 8.308945181740812e-05, "loss": 0.021506596356630325, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.02174, "step": 165, "tokens/total": 5330792, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 74999} +{"epoch": 0.6484375, "grad_norm": 0.20816871523857117, "learning_rate": 8.286207725787153e-05, "loss": 0.0024882759898900986, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00249, "step": 166, "tokens/total": 5364144, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 75444} +{"epoch": 0.65234375, "grad_norm": 0.29756858944892883, "learning_rate": 8.263354324357182e-05, "loss": 0.002563042100518942, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00257, "step": 167, "tokens/total": 5396656, "tokens/train_per_sec_per_gpu": 6.78, "tokens/trainable": 75879} +{"epoch": 0.65625, "grad_norm": 0.3119931221008301, "learning_rate": 8.240385928474219e-05, "loss": 0.0012007345212623477, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.0012, "step": 168, "tokens/total": 5427736, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 76293} +{"epoch": 0.66015625, "grad_norm": 0.6342458724975586, "learning_rate": 8.217303493946967e-05, "loss": 0.009611368179321289, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00966, "step": 169, "tokens/total": 5461752, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 76741} +{"epoch": 0.6640625, "grad_norm": 0.16293556988239288, "learning_rate": 8.194107981329746e-05, "loss": 0.0012508693616837263, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00125, "step": 170, "tokens/total": 5498688, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 77195} +{"epoch": 0.66796875, "grad_norm": 0.010820732451975346, "learning_rate": 8.170800355882518e-05, "loss": 0.00018970767268911004, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00019, "step": 171, "tokens/total": 5530936, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 77662} +{"epoch": 0.671875, "grad_norm": 0.7181956768035889, "learning_rate": 8.147381587530713e-05, "loss": 0.018970023840665817, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01915, "step": 172, "tokens/total": 5564176, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 78116} +{"epoch": 0.67578125, "grad_norm": 1.1290725469589233, "learning_rate": 8.123852650824877e-05, "loss": 0.028769204393029213, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.02919, "step": 173, "tokens/total": 5596856, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 78599} +{"epoch": 0.6796875, "grad_norm": 0.103641577064991, "learning_rate": 8.100214524900103e-05, "loss": 0.0014373691519722342, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00144, "step": 174, "tokens/total": 5630016, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 79030} +{"epoch": 0.68359375, "grad_norm": 0.45007824897766113, "learning_rate": 8.076468193435301e-05, "loss": 0.009156396612524986, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.0092, "step": 175, "tokens/total": 5659656, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 79454} +{"epoch": 0.6875, "grad_norm": 0.8728406429290771, "learning_rate": 8.052614644612253e-05, "loss": 0.01890270784497261, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.01908, "step": 176, "tokens/total": 5692408, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 79915} +{"epoch": 0.69140625, "grad_norm": 0.08347821235656738, "learning_rate": 8.028654871074489e-05, "loss": 0.0016565999248996377, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00166, "step": 177, "tokens/total": 5724672, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 80356} +{"epoch": 0.6953125, "grad_norm": 0.7281169295310974, "learning_rate": 8.004589869885986e-05, "loss": 0.03393524885177612, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.03452, "step": 178, "tokens/total": 5754384, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 80774} +{"epoch": 0.69921875, "grad_norm": 8.682642936706543, "learning_rate": 7.980420642489674e-05, "loss": 0.005955239292234182, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00597, "step": 179, "tokens/total": 5789344, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 81247} +{"epoch": 0.703125, "grad_norm": 0.3338685631752014, "learning_rate": 7.95614819466576e-05, "loss": 0.00917427521198988, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00922, "step": 180, "tokens/total": 5821816, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 81713} +{"epoch": 0.70703125, "grad_norm": 0.5563055872917175, "learning_rate": 7.931773536489872e-05, "loss": 0.01324197743088007, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.01333, "step": 181, "tokens/total": 5854168, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 82158} +{"epoch": 0.7109375, "grad_norm": 0.4151066839694977, "learning_rate": 7.907297682291035e-05, "loss": 0.01309298537671566, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01318, "step": 182, "tokens/total": 5886064, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 82621} +{"epoch": 0.71484375, "grad_norm": 0.39057162404060364, "learning_rate": 7.882721650609442e-05, "loss": 0.015397674404084682, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.01552, "step": 183, "tokens/total": 5921648, "tokens/train_per_sec_per_gpu": 6.67, "tokens/trainable": 83122} +{"epoch": 0.71875, "grad_norm": 0.2792472541332245, "learning_rate": 7.85804646415409e-05, "loss": 0.010760759003460407, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.01082, "step": 184, "tokens/total": 5954952, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 83578} +{"epoch": 0.72265625, "grad_norm": 0.18485654890537262, "learning_rate": 7.833273149760207e-05, "loss": 0.005882546305656433, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.0059, "step": 185, "tokens/total": 5985432, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 84048} +{"epoch": 0.7265625, "grad_norm": 0.5762144923210144, "learning_rate": 7.808402738346527e-05, "loss": 0.018682509660720825, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01886, "step": 186, "tokens/total": 6020664, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 84497} +{"epoch": 0.73046875, "grad_norm": 0.11986444890499115, "learning_rate": 7.783436264872382e-05, "loss": 0.003686182899400592, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00369, "step": 187, "tokens/total": 6055024, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 84983} +{"epoch": 0.734375, "grad_norm": 0.26766300201416016, "learning_rate": 7.758374768294647e-05, "loss": 0.014424724504351616, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.01453, "step": 188, "tokens/total": 6088296, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 85462} +{"epoch": 0.73828125, "grad_norm": 0.137153759598732, "learning_rate": 7.733219291524489e-05, "loss": 0.00594547251239419, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00596, "step": 189, "tokens/total": 6119840, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 85892} +{"epoch": 0.7421875, "grad_norm": 0.266046404838562, "learning_rate": 7.707970881383977e-05, "loss": 0.005898843985050917, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.00592, "step": 190, "tokens/total": 6149880, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 86341} +{"epoch": 0.74609375, "grad_norm": 0.13804011046886444, "learning_rate": 7.682630588562518e-05, "loss": 0.004384330939501524, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00439, "step": 191, "tokens/total": 6182904, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 86772} +{"epoch": 0.75, "grad_norm": 0.13118328154087067, "learning_rate": 7.657199467573129e-05, "loss": 0.003472511889412999, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00348, "step": 192, "tokens/total": 6214144, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 87244} +{"epoch": 0.75390625, "grad_norm": 0.09639117121696472, "learning_rate": 7.631678576708561e-05, "loss": 0.001059450558386743, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00106, "step": 193, "tokens/total": 6247072, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 87736} +{"epoch": 0.7578125, "grad_norm": 0.07799465954303741, "learning_rate": 7.606068977997255e-05, "loss": 0.00114087900146842, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00114, "step": 194, "tokens/total": 6281112, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 88214} +{"epoch": 0.76171875, "grad_norm": 0.636174738407135, "learning_rate": 7.580371737159148e-05, "loss": 0.006994582712650299, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00702, "step": 195, "tokens/total": 6310200, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 88625} +{"epoch": 0.765625, "grad_norm": 0.05670265108346939, "learning_rate": 7.554587923561324e-05, "loss": 0.0005631832173094153, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00056, "step": 196, "tokens/total": 6341792, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 89063} +{"epoch": 0.76953125, "grad_norm": 0.238313227891922, "learning_rate": 7.528718610173511e-05, "loss": 0.0027180369943380356, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.00272, "step": 197, "tokens/total": 6374040, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 89484} +{"epoch": 0.7734375, "grad_norm": 0.3069382309913635, "learning_rate": 7.502764873523431e-05, "loss": 0.0038760972674936056, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00388, "step": 198, "tokens/total": 6405552, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 89922} +{"epoch": 0.77734375, "grad_norm": 0.18566162884235382, "learning_rate": 7.476727793652011e-05, "loss": 0.0007992719765752554, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.0008, "step": 199, "tokens/total": 6436016, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 90401} +{"epoch": 0.78125, "grad_norm": 0.4199519157409668, "learning_rate": 7.450608454068415e-05, "loss": 0.007317767944186926, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00734, "step": 200, "tokens/total": 6468656, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 90842} +{"epoch": 0.78515625, "grad_norm": 0.0679422914981842, "learning_rate": 7.424407941704987e-05, "loss": 0.0008756555616855621, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00088, "step": 201, "tokens/total": 6502728, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 91343} +{"epoch": 0.7890625, "grad_norm": 1.1746008396148682, "learning_rate": 7.398127346871986e-05, "loss": 0.010886764153838158, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.01095, "step": 202, "tokens/total": 6535616, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 91798} +{"epoch": 0.79296875, "grad_norm": 0.008153547532856464, "learning_rate": 7.371767763212238e-05, "loss": 6.489489896921441e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00006, "step": 203, "tokens/total": 6567088, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 92251} +{"epoch": 0.796875, "grad_norm": 0.3589445948600769, "learning_rate": 7.345330287655617e-05, "loss": 0.009513335302472115, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00956, "step": 204, "tokens/total": 6600808, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 92714} +{"epoch": 0.80078125, "grad_norm": 0.22978432476520538, "learning_rate": 7.31881602037339e-05, "loss": 0.003151153912767768, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.00316, "step": 205, "tokens/total": 6631384, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 93154} +{"epoch": 0.8046875, "grad_norm": 0.2754632234573364, "learning_rate": 7.29222606473245e-05, "loss": 0.0030242244247347116, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00303, "step": 206, "tokens/total": 6661888, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 93599} +{"epoch": 0.80859375, "grad_norm": 0.08114451915025711, "learning_rate": 7.265561527249383e-05, "loss": 0.0012504898477345705, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00125, "step": 207, "tokens/total": 6693456, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 94036} +{"epoch": 0.8125, "grad_norm": 0.24816712737083435, "learning_rate": 7.238823517544436e-05, "loss": 0.007311441469937563, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00734, "step": 208, "tokens/total": 6727840, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 94506} +{"epoch": 0.81640625, "grad_norm": 0.0583488829433918, "learning_rate": 7.212013148295333e-05, "loss": 0.0005086186574772, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00051, "step": 209, "tokens/total": 6761704, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 94988} +{"epoch": 0.8203125, "grad_norm": 0.7018738389015198, "learning_rate": 7.185131535190975e-05, "loss": 0.011026815511286259, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.01109, "step": 210, "tokens/total": 6792648, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 95497} +{"epoch": 0.82421875, "grad_norm": 0.08982410281896591, "learning_rate": 7.158179796885005e-05, "loss": 0.0008654513512738049, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00087, "step": 211, "tokens/total": 6824424, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 95948} +{"epoch": 0.828125, "grad_norm": 0.010224212892353535, "learning_rate": 7.131159054949273e-05, "loss": 9.709637379273772e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.0001, "step": 212, "tokens/total": 6861216, "tokens/train_per_sec_per_gpu": 7.02, "tokens/trainable": 96422} +{"epoch": 0.83203125, "grad_norm": 0.09295199811458588, "learning_rate": 7.104070433827139e-05, "loss": 0.001050326507538557, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00105, "step": 213, "tokens/total": 6892704, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 96821} +{"epoch": 0.8359375, "grad_norm": 0.253408282995224, "learning_rate": 7.076915060786705e-05, "loss": 0.0025529649574309587, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.00256, "step": 214, "tokens/total": 6924816, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 97263} +{"epoch": 0.83984375, "grad_norm": 0.718032717704773, "learning_rate": 7.049694065873882e-05, "loss": 0.015131975524127483, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.01525, "step": 215, "tokens/total": 6957536, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 97720} +{"epoch": 0.84375, "grad_norm": 0.06457845121622086, "learning_rate": 7.022408581865382e-05, "loss": 0.001134454389102757, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00114, "step": 216, "tokens/total": 6991448, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 98194} +{"epoch": 0.84765625, "grad_norm": 0.2126772552728653, "learning_rate": 6.99505974422157e-05, "loss": 0.0021535600535571575, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00216, "step": 217, "tokens/total": 7023712, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 98667} +{"epoch": 0.8515625, "grad_norm": 0.5895292162895203, "learning_rate": 6.967648691039213e-05, "loss": 0.006399981677532196, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00642, "step": 218, "tokens/total": 7055040, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 99100} +{"epoch": 0.85546875, "grad_norm": 0.039377033710479736, "learning_rate": 6.940176563004123e-05, "loss": 0.00036051677307114005, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00036, "step": 219, "tokens/total": 7084944, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 99532} +{"epoch": 0.859375, "grad_norm": 0.44727712869644165, "learning_rate": 6.912644503343682e-05, "loss": 0.010677183978259563, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01073, "step": 220, "tokens/total": 7119264, "tokens/train_per_sec_per_gpu": 7.08, "tokens/trainable": 99970} +{"epoch": 0.86328125, "grad_norm": 0.15163901448249817, "learning_rate": 6.885053657779273e-05, "loss": 0.002698754658922553, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.0027, "step": 221, "tokens/total": 7150848, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 100442} +{"epoch": 0.8671875, "grad_norm": 0.32260656356811523, "learning_rate": 6.857405174478604e-05, "loss": 0.005796534474939108, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00581, "step": 222, "tokens/total": 7185176, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 100895} +{"epoch": 0.87109375, "grad_norm": 0.07340756803750992, "learning_rate": 6.82970020400792e-05, "loss": 0.0012123500928282738, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00121, "step": 223, "tokens/total": 7219024, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 101342} +{"epoch": 0.875, "grad_norm": 0.5398057103157043, "learning_rate": 6.801939899284132e-05, "loss": 0.006512120366096497, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00653, "step": 224, "tokens/total": 7253816, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 101848} +{"epoch": 0.87890625, "grad_norm": 0.32854601740837097, "learning_rate": 6.774125415526827e-05, "loss": 0.003290309337899089, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.0033, "step": 225, "tokens/total": 7286344, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 102319} +{"epoch": 0.8828125, "grad_norm": 0.17739935219287872, "learning_rate": 6.746257910210214e-05, "loss": 0.002539399079978466, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00254, "step": 226, "tokens/total": 7317952, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 102769} +{"epoch": 0.88671875, "grad_norm": 0.08348646014928818, "learning_rate": 6.718338543014937e-05, "loss": 0.001231455011293292, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00123, "step": 227, "tokens/total": 7347336, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 103219} +{"epoch": 0.890625, "grad_norm": 0.1548519879579544, "learning_rate": 6.69036847577983e-05, "loss": 0.002696177689358592, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.0027, "step": 228, "tokens/total": 7377592, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 103675} +{"epoch": 0.89453125, "grad_norm": 0.2983679175376892, "learning_rate": 6.662348872453553e-05, "loss": 0.0049700187519192696, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00498, "step": 229, "tokens/total": 7410184, "tokens/train_per_sec_per_gpu": 10.38, "tokens/trainable": 104136} +{"epoch": 0.8984375, "grad_norm": 0.10370627045631409, "learning_rate": 6.63428089904618e-05, "loss": 0.001885075238533318, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00189, "step": 230, "tokens/total": 7443984, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 104588} +{"epoch": 0.90234375, "grad_norm": 0.09619123488664627, "learning_rate": 6.60616572358065e-05, "loss": 0.0007874662987887859, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00079, "step": 231, "tokens/total": 7472480, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 104988} +{"epoch": 0.90625, "grad_norm": 0.1813783049583435, "learning_rate": 6.578004516044172e-05, "loss": 0.005486265756189823, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.0055, "step": 232, "tokens/total": 7505992, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 105455} +{"epoch": 0.91015625, "grad_norm": 0.19768869876861572, "learning_rate": 6.549798448339548e-05, "loss": 0.001820454141125083, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00182, "step": 233, "tokens/total": 7541168, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 105918} +{"epoch": 0.9140625, "grad_norm": 0.47815805673599243, "learning_rate": 6.521548694236384e-05, "loss": 0.011446276679635048, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.01151, "step": 234, "tokens/total": 7574208, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 106355} +{"epoch": 0.91796875, "grad_norm": 0.306344598531723, "learning_rate": 6.493256429322259e-05, "loss": 0.004568357020616531, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00458, "step": 235, "tokens/total": 7607016, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 106786} +{"epoch": 0.921875, "grad_norm": 0.940504789352417, "learning_rate": 6.464922830953799e-05, "loss": 0.02186199463903904, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.0221, "step": 236, "tokens/total": 7641744, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 107219} +{"epoch": 0.92578125, "grad_norm": 0.5791858434677124, "learning_rate": 6.436549078207688e-05, "loss": 0.015753528103232384, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.01588, "step": 237, "tokens/total": 7673096, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 107679} +{"epoch": 0.9296875, "grad_norm": 0.2143956571817398, "learning_rate": 6.408136351831592e-05, "loss": 0.0027569159865379333, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00276, "step": 238, "tokens/total": 7705344, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 108119} +{"epoch": 0.93359375, "grad_norm": 0.38503819704055786, "learning_rate": 6.379685834195036e-05, "loss": 0.00768102053552866, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00771, "step": 239, "tokens/total": 7740240, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 108626} +{"epoch": 0.9375, "grad_norm": 0.31384438276290894, "learning_rate": 6.351198709240186e-05, "loss": 0.005658402107656002, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00567, "step": 240, "tokens/total": 7775392, "tokens/train_per_sec_per_gpu": 7.01, "tokens/trainable": 109079} +{"epoch": 0.94140625, "grad_norm": 0.21000157296657562, "learning_rate": 6.32267616243259e-05, "loss": 0.003213796066120267, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00322, "step": 241, "tokens/total": 7807296, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 109542} +{"epoch": 0.9453125, "grad_norm": 0.16887682676315308, "learning_rate": 6.294119380711849e-05, "loss": 0.003200247185304761, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00321, "step": 242, "tokens/total": 7838200, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 109969} +{"epoch": 0.94921875, "grad_norm": 0.23867212235927582, "learning_rate": 6.265529552442209e-05, "loss": 0.004656102973967791, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00467, "step": 243, "tokens/total": 7870936, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 110431} +{"epoch": 0.953125, "grad_norm": 0.06348162144422531, "learning_rate": 6.236907867363127e-05, "loss": 0.0007496631005778909, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00075, "step": 244, "tokens/total": 7902512, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 110877} +{"epoch": 0.95703125, "grad_norm": 0.2161974012851715, "learning_rate": 6.208255516539749e-05, "loss": 0.0008841030648909509, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00088, "step": 245, "tokens/total": 7933224, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 111350} +{"epoch": 0.9609375, "grad_norm": 0.3220353126525879, "learning_rate": 6.179573692313344e-05, "loss": 0.011239217594265938, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.0113, "step": 246, "tokens/total": 7968456, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 111798} +{"epoch": 0.96484375, "grad_norm": 0.17744863033294678, "learning_rate": 6.150863588251694e-05, "loss": 0.002387469168752432, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00239, "step": 247, "tokens/total": 8003592, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 112264} +{"epoch": 0.96875, "grad_norm": 0.3190135061740875, "learning_rate": 6.122126399099419e-05, "loss": 0.0038086040876805782, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00382, "step": 248, "tokens/total": 8040048, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 112772} +{"epoch": 0.97265625, "grad_norm": 0.8493856191635132, "learning_rate": 6.0933633207282615e-05, "loss": 0.003439623164013028, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00345, "step": 249, "tokens/total": 8074000, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 113278} +{"epoch": 0.9765625, "grad_norm": 0.3094353675842285, "learning_rate": 6.064575550087316e-05, "loss": 0.006200592499226332, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00622, "step": 250, "tokens/total": 8108304, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 113716} +{"epoch": 0.98046875, "grad_norm": 0.10057081282138824, "learning_rate": 6.0357642851532245e-05, "loss": 0.0023898479994386435, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00239, "step": 251, "tokens/total": 8143464, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 114207} +{"epoch": 0.984375, "grad_norm": 0.28853628039360046, "learning_rate": 6.0069307248803294e-05, "loss": 0.004067230969667435, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00408, "step": 252, "tokens/total": 8175576, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 114628} +{"epoch": 0.98828125, "grad_norm": 0.016048744320869446, "learning_rate": 5.9780760691507635e-05, "loss": 0.00033024040749296546, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00033, "step": 253, "tokens/total": 8205528, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 115060} +{"epoch": 0.9921875, "grad_norm": 0.3392256796360016, "learning_rate": 5.9492015187245334e-05, "loss": 0.0069381664507091045, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00696, "step": 254, "tokens/total": 8239424, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 115514} +{"epoch": 0.99609375, "grad_norm": 0.10022246837615967, "learning_rate": 5.920308275189541e-05, "loss": 0.0009155977750197053, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00092, "step": 255, "tokens/total": 8270392, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 115915} +{"epoch": 1.0, "grad_norm": 0.402020663022995, "learning_rate": 5.8913975409115874e-05, "loss": 0.005164752248674631, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00518, "step": 256, "tokens/total": 8302096, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 116378} +{"epoch": 1.00390625, "grad_norm": 0.346927672624588, "learning_rate": 5.8624705189843395e-05, "loss": 0.004235787782818079, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00424, "step": 257, "tokens/total": 8337064, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 116865} +{"epoch": 1.0078125, "grad_norm": 0.18174701929092407, "learning_rate": 5.833528413179249e-05, "loss": 0.0027485904283821583, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00275, "step": 258, "tokens/total": 8372840, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 117363} +{"epoch": 1.01171875, "grad_norm": 0.27459341287612915, "learning_rate": 5.80457242789548e-05, "loss": 0.005203673150390387, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00522, "step": 259, "tokens/total": 8409728, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 117860} +{"epoch": 1.015625, "grad_norm": 0.2085372358560562, "learning_rate": 5.77560376810977e-05, "loss": 0.0022677797824144363, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.00227, "step": 260, "tokens/total": 8442296, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 118324} +{"epoch": 1.01953125, "grad_norm": 0.09460653364658356, "learning_rate": 5.7466236393263005e-05, "loss": 0.000590414390899241, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00059, "step": 261, "tokens/total": 8474568, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 118770} +{"epoch": 1.0234375, "grad_norm": 0.010687476955354214, "learning_rate": 5.717633247526522e-05, "loss": 0.00013250473421067, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00013, "step": 262, "tokens/total": 8505520, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 119206} +{"epoch": 1.02734375, "grad_norm": 0.08002200722694397, "learning_rate": 5.688633799118971e-05, "loss": 0.0008579186396673322, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00086, "step": 263, "tokens/total": 8535064, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 119610} +{"epoch": 1.03125, "grad_norm": 0.060075804591178894, "learning_rate": 5.659626500889066e-05, "loss": 0.0009103114134632051, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00091, "step": 264, "tokens/total": 8563600, "tokens/train_per_sec_per_gpu": 6.94, "tokens/trainable": 120080} +{"epoch": 1.03515625, "grad_norm": 0.026180358603596687, "learning_rate": 5.6306125599488905e-05, "loss": 0.0002846449497155845, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00028, "step": 265, "tokens/total": 8592648, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 120519} +{"epoch": 1.0390625, "grad_norm": 0.1381753832101822, "learning_rate": 5.601593183686955e-05, "loss": 0.0009789905743673444, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00098, "step": 266, "tokens/total": 8627328, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 120995} +{"epoch": 1.04296875, "grad_norm": 0.1549297273159027, "learning_rate": 5.572569579717961e-05, "loss": 0.0014890378806740046, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00149, "step": 267, "tokens/total": 8659760, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 121459} +{"epoch": 1.046875, "grad_norm": 0.007607798092067242, "learning_rate": 5.543542955832538e-05, "loss": 0.00010280764399794862, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.0001, "step": 268, "tokens/total": 8696968, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 121902} +{"epoch": 1.05078125, "grad_norm": 0.17960073053836823, "learning_rate": 5.514514519946986e-05, "loss": 0.002102939411997795, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00211, "step": 269, "tokens/total": 8735392, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 122399} +{"epoch": 1.0546875, "grad_norm": 0.05522049218416214, "learning_rate": 5.485485480053015e-05, "loss": 0.0002548767370171845, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00025, "step": 270, "tokens/total": 8769840, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 122804} +{"epoch": 1.05859375, "grad_norm": 0.013404260389506817, "learning_rate": 5.4564570441674645e-05, "loss": 0.00013809502706862986, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00014, "step": 271, "tokens/total": 8800648, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 123244} +{"epoch": 1.0625, "grad_norm": 0.021674761548638344, "learning_rate": 5.42743042028204e-05, "loss": 0.00022332070511765778, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00022, "step": 272, "tokens/total": 8832480, "tokens/train_per_sec_per_gpu": 6.7, "tokens/trainable": 123683} +{"epoch": 1.06640625, "grad_norm": 0.05220451578497887, "learning_rate": 5.3984068163130464e-05, "loss": 0.0004111276939511299, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00041, "step": 273, "tokens/total": 8864840, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 124153} +{"epoch": 1.0703125, "grad_norm": 0.050893474370241165, "learning_rate": 5.369387440051111e-05, "loss": 0.00046427358756773174, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00046, "step": 274, "tokens/total": 8897272, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 124616} +{"epoch": 1.07421875, "grad_norm": 0.14482945203781128, "learning_rate": 5.340373499110935e-05, "loss": 0.001527882064692676, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00153, "step": 275, "tokens/total": 8930688, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 125109} +{"epoch": 1.078125, "grad_norm": 0.013165043666958809, "learning_rate": 5.3113662008810304e-05, "loss": 7.148133590817451e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00007, "step": 276, "tokens/total": 8967216, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 125591} +{"epoch": 1.08203125, "grad_norm": 0.16358523070812225, "learning_rate": 5.282366752473479e-05, "loss": 0.0008693077834323049, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00087, "step": 277, "tokens/total": 9001368, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 126065} +{"epoch": 1.0859375, "grad_norm": 0.013984696939587593, "learning_rate": 5.2533763606737005e-05, "loss": 8.372303273063153e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00008, "step": 278, "tokens/total": 9036808, "tokens/train_per_sec_per_gpu": 6.8, "tokens/trainable": 126480} +{"epoch": 1.08984375, "grad_norm": 0.020089823752641678, "learning_rate": 5.224396231890232e-05, "loss": 0.00011668707884382457, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00012, "step": 279, "tokens/total": 9067840, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 126929} +{"epoch": 1.09375, "grad_norm": 0.005265984684228897, "learning_rate": 5.195427572104522e-05, "loss": 6.323924753814936e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00006, "step": 280, "tokens/total": 9098936, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 127366} +{"epoch": 1.09765625, "grad_norm": 0.0016551823355257511, "learning_rate": 5.166471586820751e-05, "loss": 2.212479193985928e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00002, "step": 281, "tokens/total": 9133264, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 127831} +{"epoch": 1.1015625, "grad_norm": 0.01970026083290577, "learning_rate": 5.1375294810156615e-05, "loss": 0.00014639650180470198, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00015, "step": 282, "tokens/total": 9165008, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 128278} +{"epoch": 1.10546875, "grad_norm": 0.017014963552355766, "learning_rate": 5.1086024590884144e-05, "loss": 8.067794988164678e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00008, "step": 283, "tokens/total": 9197952, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 128727} +{"epoch": 1.109375, "grad_norm": 0.7087263464927673, "learning_rate": 5.079691724810461e-05, "loss": 0.004558033309876919, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00457, "step": 284, "tokens/total": 9232576, "tokens/train_per_sec_per_gpu": 7.01, "tokens/trainable": 129162} +{"epoch": 1.11328125, "grad_norm": 0.0047281780280172825, "learning_rate": 5.0507984812754684e-05, "loss": 4.6603770897490904e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00005, "step": 285, "tokens/total": 9264568, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 129659} +{"epoch": 1.1171875, "grad_norm": 0.3596445918083191, "learning_rate": 5.021923930849237e-05, "loss": 0.003092394210398197, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.0031, "step": 286, "tokens/total": 9296544, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 130145} +{"epoch": 1.12109375, "grad_norm": 0.22918644547462463, "learning_rate": 4.99306927511967e-05, "loss": 0.0014113991055637598, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00141, "step": 287, "tokens/total": 9327936, "tokens/train_per_sec_per_gpu": 10.27, "tokens/trainable": 130585} +{"epoch": 1.125, "grad_norm": 0.03715021163225174, "learning_rate": 4.964235714846775e-05, "loss": 0.0002000659005716443, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.0002, "step": 288, "tokens/total": 9357984, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 131005} +{"epoch": 1.12890625, "grad_norm": 0.03898895904421806, "learning_rate": 4.9354244499126866e-05, "loss": 0.00030252194846980274, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.0003, "step": 289, "tokens/total": 9388664, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 131471} +{"epoch": 1.1328125, "grad_norm": 0.024284282699227333, "learning_rate": 4.90663667927174e-05, "loss": 0.00010414862481411546, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.0001, "step": 290, "tokens/total": 9420504, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 131920} +{"epoch": 1.13671875, "grad_norm": 0.024318836629390717, "learning_rate": 4.877873600900581e-05, "loss": 0.00020397666958160698, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.0002, "step": 291, "tokens/total": 9452664, "tokens/train_per_sec_per_gpu": 6.25, "tokens/trainable": 132379} +{"epoch": 1.140625, "grad_norm": 1.3578758239746094, "learning_rate": 4.849136411748306e-05, "loss": 0.009357776492834091, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.0094, "step": 292, "tokens/total": 9488896, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 132851} +{"epoch": 1.14453125, "grad_norm": 0.003538778517395258, "learning_rate": 4.8204263076866574e-05, "loss": 1.2760663594235666e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00001, "step": 293, "tokens/total": 9522112, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 133351} +{"epoch": 1.1484375, "grad_norm": 0.5009358525276184, "learning_rate": 4.791744483460251e-05, "loss": 0.004583973437547684, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00459, "step": 294, "tokens/total": 9555632, "tokens/train_per_sec_per_gpu": 6.5, "tokens/trainable": 133784} +{"epoch": 1.15234375, "grad_norm": 0.00210201577283442, "learning_rate": 4.7630921326368736e-05, "loss": 1.660883026488591e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00002, "step": 295, "tokens/total": 9590904, "tokens/train_per_sec_per_gpu": 6.33, "tokens/trainable": 134209} +{"epoch": 1.15625, "grad_norm": 0.2311408668756485, "learning_rate": 4.7344704475577916e-05, "loss": 0.0013564643450081348, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00136, "step": 296, "tokens/total": 9620288, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 134645} +{"epoch": 1.16015625, "grad_norm": 0.1525583416223526, "learning_rate": 4.705880619288153e-05, "loss": 0.0014267867663875222, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00143, "step": 297, "tokens/total": 9651984, "tokens/train_per_sec_per_gpu": 6.98, "tokens/trainable": 135047} +{"epoch": 1.1640625, "grad_norm": 0.6566368937492371, "learning_rate": 4.677323837567412e-05, "loss": 0.012647565454244614, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.01273, "step": 298, "tokens/total": 9681368, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 135474} +{"epoch": 1.16796875, "grad_norm": 0.02466162107884884, "learning_rate": 4.6488012907598146e-05, "loss": 4.32615343015641e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00004, "step": 299, "tokens/total": 9712872, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 135948} +{"epoch": 1.171875, "grad_norm": 0.15910381078720093, "learning_rate": 4.620314165804964e-05, "loss": 0.0014094787184149027, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00141, "step": 300, "tokens/total": 9745184, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 136421} +{"epoch": 1.17578125, "grad_norm": 0.06796476244926453, "learning_rate": 4.591863648168407e-05, "loss": 0.00033367431024089456, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00033, "step": 301, "tokens/total": 9778928, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 136886} +{"epoch": 1.1796875, "grad_norm": 0.003200130071491003, "learning_rate": 4.5634509217923135e-05, "loss": 2.7204376237932593e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00003, "step": 302, "tokens/total": 9810752, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 137321} +{"epoch": 1.18359375, "grad_norm": 0.270062118768692, "learning_rate": 4.535077169046201e-05, "loss": 0.006773650646209717, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.0068, "step": 303, "tokens/total": 9843672, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 137758} +{"epoch": 1.1875, "grad_norm": 0.11059747636318207, "learning_rate": 4.506743570677743e-05, "loss": 0.0004602503322530538, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00046, "step": 304, "tokens/total": 9873752, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 138190} +{"epoch": 1.19140625, "grad_norm": 0.03279528766870499, "learning_rate": 4.478451305763618e-05, "loss": 0.00021502605522982776, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00022, "step": 305, "tokens/total": 9906272, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 138650} +{"epoch": 1.1953125, "grad_norm": 0.16229885816574097, "learning_rate": 4.450201551660454e-05, "loss": 0.0021749797742813826, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00218, "step": 306, "tokens/total": 9936296, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 139082} +{"epoch": 1.19921875, "grad_norm": 0.010881377384066582, "learning_rate": 4.4219954839558276e-05, "loss": 9.774251520866528e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0001, "step": 307, "tokens/total": 9967720, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 139523} +{"epoch": 1.203125, "grad_norm": 1.3459503650665283, "learning_rate": 4.393834276419352e-05, "loss": 0.019687402993440628, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.01988, "step": 308, "tokens/total": 9997176, "tokens/train_per_sec_per_gpu": 10.64, "tokens/trainable": 140010} +{"epoch": 1.20703125, "grad_norm": 0.2526399791240692, "learning_rate": 4.36571910095382e-05, "loss": 0.004456609487533569, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00447, "step": 309, "tokens/total": 10031872, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 140454} +{"epoch": 1.2109375, "grad_norm": 0.5763605833053589, "learning_rate": 4.337651127546448e-05, "loss": 0.010862095281481743, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01092, "step": 310, "tokens/total": 10065240, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 140905} +{"epoch": 1.21484375, "grad_norm": 0.20384743809700012, "learning_rate": 4.3096315242201736e-05, "loss": 0.0025801467709243298, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00258, "step": 311, "tokens/total": 10096640, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 141339} +{"epoch": 1.21875, "grad_norm": 0.02705318108201027, "learning_rate": 4.2816614569850635e-05, "loss": 0.00019265526498202235, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00019, "step": 312, "tokens/total": 10130192, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 141778} +{"epoch": 1.22265625, "grad_norm": 0.5599064230918884, "learning_rate": 4.2537420897897864e-05, "loss": 0.006265648175030947, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00629, "step": 313, "tokens/total": 10164248, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 142255} +{"epoch": 1.2265625, "grad_norm": 0.6140104532241821, "learning_rate": 4.225874584473174e-05, "loss": 0.006291563156992197, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00631, "step": 314, "tokens/total": 10194664, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 142693} +{"epoch": 1.23046875, "grad_norm": 0.051254794001579285, "learning_rate": 4.19806010071587e-05, "loss": 0.0013036055024713278, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.0013, "step": 315, "tokens/total": 10227768, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 143157} +{"epoch": 1.234375, "grad_norm": 0.013171689584851265, "learning_rate": 4.170299795992081e-05, "loss": 0.00026905519189313054, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00027, "step": 316, "tokens/total": 10260464, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 143603} +{"epoch": 1.23828125, "grad_norm": 0.04427060857415199, "learning_rate": 4.142594825521398e-05, "loss": 0.0007526755216531456, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00075, "step": 317, "tokens/total": 10297528, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 144084} +{"epoch": 1.2421875, "grad_norm": 0.29741138219833374, "learning_rate": 4.114946342220728e-05, "loss": 0.0024511555675417185, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00245, "step": 318, "tokens/total": 10331448, "tokens/train_per_sec_per_gpu": 6.88, "tokens/trainable": 144532} +{"epoch": 1.24609375, "grad_norm": 0.07959617674350739, "learning_rate": 4.087355496656321e-05, "loss": 0.0011619477299973369, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00116, "step": 319, "tokens/total": 10363152, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 145004} +{"epoch": 1.25, "grad_norm": 0.1726681888103485, "learning_rate": 4.05982343699588e-05, "loss": 0.0013723218580707908, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00137, "step": 320, "tokens/total": 10397832, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 145498} +{"epoch": 1.25390625, "grad_norm": 0.4645351469516754, "learning_rate": 4.0323513089607876e-05, "loss": 0.002733728149905801, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00274, "step": 321, "tokens/total": 10431416, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 145965} +{"epoch": 1.2578125, "grad_norm": 0.16596196591854095, "learning_rate": 4.004940255778431e-05, "loss": 0.0005208267248235643, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00052, "step": 322, "tokens/total": 10467384, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 146409} +{"epoch": 1.26171875, "grad_norm": 0.03597233071923256, "learning_rate": 3.977591418134619e-05, "loss": 0.0005224198685027659, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00052, "step": 323, "tokens/total": 10500792, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 146849} +{"epoch": 1.265625, "grad_norm": 0.24593482911586761, "learning_rate": 3.95030593412612e-05, "loss": 0.0018552937544882298, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00186, "step": 324, "tokens/total": 10532592, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 147272} +{"epoch": 1.26953125, "grad_norm": 0.04727784916758537, "learning_rate": 3.923084939213296e-05, "loss": 0.00014843512326478958, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00015, "step": 325, "tokens/total": 10562392, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 147694} +{"epoch": 1.2734375, "grad_norm": 0.04753721505403519, "learning_rate": 3.895929566172861e-05, "loss": 0.0004917416372336447, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00049, "step": 326, "tokens/total": 10595416, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 148158} +{"epoch": 1.27734375, "grad_norm": 0.2382982224225998, "learning_rate": 3.868840945050728e-05, "loss": 0.0008919517858885229, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00089, "step": 327, "tokens/total": 10630504, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 148590} +{"epoch": 1.28125, "grad_norm": 0.010510619729757309, "learning_rate": 3.841820203114995e-05, "loss": 0.00018061036826111376, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00018, "step": 328, "tokens/total": 10661632, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 149037} +{"epoch": 1.28515625, "grad_norm": 0.1694512516260147, "learning_rate": 3.814868464809027e-05, "loss": 0.0009956283029168844, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.001, "step": 329, "tokens/total": 10691544, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 149439} +{"epoch": 1.2890625, "grad_norm": 0.029652971774339676, "learning_rate": 3.787986851704667e-05, "loss": 0.00024111296806950122, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00024, "step": 330, "tokens/total": 10724768, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 149869} +{"epoch": 1.29296875, "grad_norm": 0.016415398567914963, "learning_rate": 3.7611764824555654e-05, "loss": 0.00018863605509977788, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00019, "step": 331, "tokens/total": 10756008, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 150343} +{"epoch": 1.296875, "grad_norm": 0.2350711077451706, "learning_rate": 3.734438472750619e-05, "loss": 0.0022888039238750935, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00229, "step": 332, "tokens/total": 10793152, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 150814} +{"epoch": 1.30078125, "grad_norm": 0.016515210270881653, "learning_rate": 3.707773935267552e-05, "loss": 0.0003484345506876707, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00035, "step": 333, "tokens/total": 10827624, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 151300} +{"epoch": 1.3046875, "grad_norm": 0.004524084739387035, "learning_rate": 3.68118397962661e-05, "loss": 9.207394759869203e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00009, "step": 334, "tokens/total": 10859712, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 151708} +{"epoch": 1.30859375, "grad_norm": 0.1359630674123764, "learning_rate": 3.654669712344384e-05, "loss": 0.0009931721724569798, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00099, "step": 335, "tokens/total": 10888792, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 152164} +{"epoch": 1.3125, "grad_norm": 0.002942292485386133, "learning_rate": 3.628232236787763e-05, "loss": 5.5730393796693534e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00006, "step": 336, "tokens/total": 10922968, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 152603} +{"epoch": 1.31640625, "grad_norm": 0.061009202152490616, "learning_rate": 3.6018726531280144e-05, "loss": 0.00022504512162413448, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00023, "step": 337, "tokens/total": 10958000, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 153094} +{"epoch": 1.3203125, "grad_norm": 0.36157262325286865, "learning_rate": 3.575592058295017e-05, "loss": 0.0029381001368165016, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00294, "step": 338, "tokens/total": 10989872, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 153573} +{"epoch": 1.32421875, "grad_norm": 0.0008127546170726418, "learning_rate": 3.549391545931585e-05, "loss": 1.8556256691226736e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00002, "step": 339, "tokens/total": 11020784, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 154033} +{"epoch": 1.328125, "grad_norm": 0.15152356028556824, "learning_rate": 3.5232722063479914e-05, "loss": 0.0023998385295271873, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.0024, "step": 340, "tokens/total": 11053240, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 154482} +{"epoch": 1.33203125, "grad_norm": 0.5677337050437927, "learning_rate": 3.49723512647657e-05, "loss": 0.002476363442838192, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00248, "step": 341, "tokens/total": 11089040, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 154982} +{"epoch": 1.3359375, "grad_norm": 0.030063161626458168, "learning_rate": 3.471281389826491e-05, "loss": 0.0002790256403386593, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00028, "step": 342, "tokens/total": 11119672, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 155387} +{"epoch": 1.33984375, "grad_norm": 0.20980821549892426, "learning_rate": 3.4454120764386764e-05, "loss": 0.000629740534350276, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00063, "step": 343, "tokens/total": 11155040, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 155842} +{"epoch": 1.34375, "grad_norm": 0.2075635939836502, "learning_rate": 3.4196282628408526e-05, "loss": 0.0020950171165168285, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.0021, "step": 344, "tokens/total": 11185952, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 156287} +{"epoch": 1.34765625, "grad_norm": 0.06063588336110115, "learning_rate": 3.3939310220027456e-05, "loss": 0.0005065551958978176, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00051, "step": 345, "tokens/total": 11219272, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 156782} +{"epoch": 1.3515625, "grad_norm": 0.004037253558635712, "learning_rate": 3.3683214232914404e-05, "loss": 2.515935557312332e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00003, "step": 346, "tokens/total": 11253424, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 157255} +{"epoch": 1.35546875, "grad_norm": 0.1053890511393547, "learning_rate": 3.342800532426873e-05, "loss": 0.001088283839635551, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00109, "step": 347, "tokens/total": 11286216, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 157721} +{"epoch": 1.359375, "grad_norm": 0.02458302676677704, "learning_rate": 3.317369411437484e-05, "loss": 0.00040697265649214387, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00041, "step": 348, "tokens/total": 11319480, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 158189} +{"epoch": 1.36328125, "grad_norm": 0.02816951647400856, "learning_rate": 3.292029118616024e-05, "loss": 0.00021252260194160044, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00021, "step": 349, "tokens/total": 11350472, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 158612} +{"epoch": 1.3671875, "grad_norm": 0.07273983210325241, "learning_rate": 3.266780708475511e-05, "loss": 0.0008935079094953835, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00089, "step": 350, "tokens/total": 11383088, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 159090} +{"epoch": 1.37109375, "grad_norm": 0.005811078008264303, "learning_rate": 3.241625231705354e-05, "loss": 3.680619192891754e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00004, "step": 351, "tokens/total": 11418392, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 159549} +{"epoch": 1.375, "grad_norm": 0.010159499011933804, "learning_rate": 3.216563735127618e-05, "loss": 0.0001383040944347158, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00014, "step": 352, "tokens/total": 11449784, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 160026} +{"epoch": 1.37890625, "grad_norm": 0.03738563135266304, "learning_rate": 3.191597261653475e-05, "loss": 0.0003248432476539165, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00032, "step": 353, "tokens/total": 11484240, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 160487} +{"epoch": 1.3828125, "grad_norm": 0.012140362523496151, "learning_rate": 3.166726850239794e-05, "loss": 8.11247227829881e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00008, "step": 354, "tokens/total": 11513360, "tokens/train_per_sec_per_gpu": 10.84, "tokens/trainable": 160935} +{"epoch": 1.38671875, "grad_norm": 0.0010725450702011585, "learning_rate": 3.141953535845912e-05, "loss": 2.0377447071950883e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00002, "step": 355, "tokens/total": 11543112, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 161385} +{"epoch": 1.390625, "grad_norm": 0.044404856860637665, "learning_rate": 3.11727834939056e-05, "loss": 0.0004402322228997946, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00044, "step": 356, "tokens/total": 11573464, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 161849} +{"epoch": 1.39453125, "grad_norm": 0.20625321567058563, "learning_rate": 3.092702317708967e-05, "loss": 0.003790990449488163, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.0038, "step": 357, "tokens/total": 11604896, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 162306} +{"epoch": 1.3984375, "grad_norm": 0.008251230232417583, "learning_rate": 3.0682264635101276e-05, "loss": 6.164339720271528e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00006, "step": 358, "tokens/total": 11636368, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 162759} +{"epoch": 1.40234375, "grad_norm": 0.002047955524176359, "learning_rate": 3.0438518053342407e-05, "loss": 2.174745714000892e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00002, "step": 359, "tokens/total": 11670216, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 163230} +{"epoch": 1.40625, "grad_norm": 0.5533941388130188, "learning_rate": 3.0195793575103266e-05, "loss": 0.0016561913071200252, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00166, "step": 360, "tokens/total": 11707816, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 163694} +{"epoch": 1.41015625, "grad_norm": 0.021619638428092003, "learning_rate": 2.9954101301140146e-05, "loss": 0.00019856019935104996, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.0002, "step": 361, "tokens/total": 11742584, "tokens/train_per_sec_per_gpu": 6.81, "tokens/trainable": 164144} +{"epoch": 1.4140625, "grad_norm": 0.14465785026550293, "learning_rate": 2.9713451289255123e-05, "loss": 0.0012402702122926712, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00124, "step": 362, "tokens/total": 11770384, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 164585} +{"epoch": 1.41796875, "grad_norm": 0.04064737260341644, "learning_rate": 2.9473853553877484e-05, "loss": 0.0003192613658029586, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00032, "step": 363, "tokens/total": 11803208, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 165059} +{"epoch": 1.421875, "grad_norm": 0.03186029568314552, "learning_rate": 2.9235318065647e-05, "loss": 0.00012663348752539605, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00013, "step": 364, "tokens/total": 11840480, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 165519} +{"epoch": 1.42578125, "grad_norm": 0.0005485680303536355, "learning_rate": 2.8997854750998964e-05, "loss": 1.254516882909229e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00001, "step": 365, "tokens/total": 11875544, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 165984} +{"epoch": 1.4296875, "grad_norm": 0.007814321666955948, "learning_rate": 2.8761473491751258e-05, "loss": 4.134769187658094e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00004, "step": 366, "tokens/total": 11907656, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 166419} +{"epoch": 1.43359375, "grad_norm": 0.008683409541845322, "learning_rate": 2.8526184124692883e-05, "loss": 5.8030698710354045e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00006, "step": 367, "tokens/total": 11941904, "tokens/train_per_sec_per_gpu": 6.32, "tokens/trainable": 166857} +{"epoch": 1.4375, "grad_norm": 0.2628297507762909, "learning_rate": 2.829199644117484e-05, "loss": 0.0021863472647964954, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00219, "step": 368, "tokens/total": 11973624, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 167267} +{"epoch": 1.44140625, "grad_norm": 0.8595948815345764, "learning_rate": 2.8058920186702553e-05, "loss": 0.011914265342056751, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.01199, "step": 369, "tokens/total": 12010360, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 167740} +{"epoch": 1.4453125, "grad_norm": 0.012932764366269112, "learning_rate": 2.782696506053033e-05, "loss": 0.00012269828584976494, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00012, "step": 370, "tokens/total": 12044544, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 168172} +{"epoch": 1.44921875, "grad_norm": 0.09176141768693924, "learning_rate": 2.7596140715257824e-05, "loss": 0.00030429454636760056, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.0003, "step": 371, "tokens/total": 12075792, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 168637} +{"epoch": 1.453125, "grad_norm": 0.00443705590441823, "learning_rate": 2.7366456756428184e-05, "loss": 6.163607758935541e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00006, "step": 372, "tokens/total": 12108088, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 169094} +{"epoch": 1.45703125, "grad_norm": 0.0033090333454310894, "learning_rate": 2.7137922742128486e-05, "loss": 3.615255991462618e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00004, "step": 373, "tokens/total": 12140888, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 169593} +{"epoch": 1.4609375, "grad_norm": 0.5774486660957336, "learning_rate": 2.691054818259188e-05, "loss": 0.0016395502025261521, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00164, "step": 374, "tokens/total": 12174560, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 170066} +{"epoch": 1.46484375, "grad_norm": 0.011007653549313545, "learning_rate": 2.6684342539801933e-05, "loss": 0.00012380506086628884, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00012, "step": 375, "tokens/total": 12204848, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 170518} +{"epoch": 1.46875, "grad_norm": 0.31458449363708496, "learning_rate": 2.645931522709877e-05, "loss": 0.00613984651863575, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00616, "step": 376, "tokens/total": 12238472, "tokens/train_per_sec_per_gpu": 10.6, "tokens/trainable": 170998} +{"epoch": 1.47265625, "grad_norm": 0.07144047319889069, "learning_rate": 2.6235475608787365e-05, "loss": 0.0003596197348088026, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00036, "step": 377, "tokens/total": 12271024, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 171459} +{"epoch": 1.4765625, "grad_norm": 0.11941705644130707, "learning_rate": 2.6012832999747916e-05, "loss": 0.0009450147626921535, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00095, "step": 378, "tokens/total": 12301312, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 171905} +{"epoch": 1.48046875, "grad_norm": 0.4655081629753113, "learning_rate": 2.579139666504821e-05, "loss": 0.014748958870768547, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01486, "step": 379, "tokens/total": 12337072, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 172351} +{"epoch": 1.484375, "grad_norm": 0.004370131064206362, "learning_rate": 2.557117581955798e-05, "loss": 2.7002486604033038e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00003, "step": 380, "tokens/total": 12368000, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 172817} +{"epoch": 1.48828125, "grad_norm": 0.26066577434539795, "learning_rate": 2.5352179627565532e-05, "loss": 0.002077270532026887, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00208, "step": 381, "tokens/total": 12399880, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 173258} +{"epoch": 1.4921875, "grad_norm": 0.0010491569992154837, "learning_rate": 2.5134417202396277e-05, "loss": 2.1359583115554415e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00002, "step": 382, "tokens/total": 12432288, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 173665} +{"epoch": 1.49609375, "grad_norm": 0.02474694326519966, "learning_rate": 2.491789760603361e-05, "loss": 0.00016240522381849587, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00016, "step": 383, "tokens/total": 12465592, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 174095} +{"epoch": 1.5, "grad_norm": 0.05742138251662254, "learning_rate": 2.4702629848741764e-05, "loss": 0.0003901042218785733, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00039, "step": 384, "tokens/total": 12498200, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 174550} +{"epoch": 1.50390625, "grad_norm": 0.013338527642190456, "learning_rate": 2.4488622888690785e-05, "loss": 0.00019060529302805662, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00019, "step": 385, "tokens/total": 12530280, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 174990} +{"epoch": 1.5078125, "grad_norm": 0.004786090925335884, "learning_rate": 2.427588563158384e-05, "loss": 6.937343277968466e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00007, "step": 386, "tokens/total": 12563360, "tokens/train_per_sec_per_gpu": 7.08, "tokens/trainable": 175476} +{"epoch": 1.51171875, "grad_norm": 0.032621223479509354, "learning_rate": 2.406442693028651e-05, "loss": 6.69650689815171e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00007, "step": 387, "tokens/total": 12593032, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 175897} +{"epoch": 1.515625, "grad_norm": 0.030467674136161804, "learning_rate": 2.3854255584458547e-05, "loss": 0.00013411043619271368, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00013, "step": 388, "tokens/total": 12628272, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 176324} +{"epoch": 1.51953125, "grad_norm": 0.0034954685252159834, "learning_rate": 2.3645380340187508e-05, "loss": 5.600021904683672e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00006, "step": 389, "tokens/total": 12661368, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 176805} +{"epoch": 1.5234375, "grad_norm": 0.031226683408021927, "learning_rate": 2.3437809889624914e-05, "loss": 0.0005565627361647785, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.77, "memory/max_allocated (GiB)": 63.77, "ppl": 1.00056, "step": 390, "tokens/total": 12691168, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 177256} +{"epoch": 1.52734375, "grad_norm": 0.007537336554378271, "learning_rate": 2.3231552870624487e-05, "loss": 9.41026519285515e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00009, "step": 391, "tokens/total": 12721264, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 177682} +{"epoch": 1.53125, "grad_norm": 0.038931991904973984, "learning_rate": 2.3026617866382657e-05, "loss": 0.00024824836873449385, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00025, "step": 392, "tokens/total": 12753640, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 178179} +{"epoch": 1.53515625, "grad_norm": 0.02545071206986904, "learning_rate": 2.2823013405081507e-05, "loss": 0.00018810218898579478, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00019, "step": 393, "tokens/total": 12787056, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 178676} +{"epoch": 1.5390625, "grad_norm": 0.0013253048527985811, "learning_rate": 2.2620747959533722e-05, "loss": 1.937313390953932e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00002, "step": 394, "tokens/total": 12819312, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 179173} +{"epoch": 1.54296875, "grad_norm": 0.004288746044039726, "learning_rate": 2.2419829946830123e-05, "loss": 5.076363595435396e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00005, "step": 395, "tokens/total": 12853376, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 179624} +{"epoch": 1.546875, "grad_norm": 0.0752938911318779, "learning_rate": 2.2220267727989325e-05, "loss": 0.0004333654942456633, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00043, "step": 396, "tokens/total": 12882960, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 180078} +{"epoch": 1.55078125, "grad_norm": 0.014231957495212555, "learning_rate": 2.202206960760984e-05, "loss": 0.00017483733245171607, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00017, "step": 397, "tokens/total": 12917016, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 180585} +{"epoch": 1.5546875, "grad_norm": 0.045220714062452316, "learning_rate": 2.182524383352446e-05, "loss": 0.0003231028968002647, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00032, "step": 398, "tokens/total": 12950056, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 181039} +{"epoch": 1.55859375, "grad_norm": 0.3974483013153076, "learning_rate": 2.1629798596457056e-05, "loss": 0.011250446550548077, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.01131, "step": 399, "tokens/total": 12981728, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 181489} +{"epoch": 1.5625, "grad_norm": 0.09761133044958115, "learning_rate": 2.1435742029681725e-05, "loss": 0.0003525603096932173, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00035, "step": 400, "tokens/total": 13011464, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 181906} +{"epoch": 1.56640625, "grad_norm": 0.11131881177425385, "learning_rate": 2.124308220868431e-05, "loss": 0.00045010869507677853, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00045, "step": 401, "tokens/total": 13044432, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 182361} +{"epoch": 1.5703125, "grad_norm": 0.04382726922631264, "learning_rate": 2.105182715082638e-05, "loss": 0.00031007296638563275, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00031, "step": 402, "tokens/total": 13075608, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 182838} +{"epoch": 1.57421875, "grad_norm": 0.031527578830718994, "learning_rate": 2.0861984815011552e-05, "loss": 0.00015611246635671705, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00016, "step": 403, "tokens/total": 13107352, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 183269} +{"epoch": 1.578125, "grad_norm": 0.0036595279816538095, "learning_rate": 2.0673563101354323e-05, "loss": 7.069985440466553e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00007, "step": 404, "tokens/total": 13140736, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 183699} +{"epoch": 1.58203125, "grad_norm": 0.22644200921058655, "learning_rate": 2.0486569850851317e-05, "loss": 0.0027483291923999786, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00275, "step": 405, "tokens/total": 13174040, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 184162} +{"epoch": 1.5859375, "grad_norm": 0.01147377211600542, "learning_rate": 2.0301012845054956e-05, "loss": 0.00011655253183562309, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00012, "step": 406, "tokens/total": 13207232, "tokens/train_per_sec_per_gpu": 6.5, "tokens/trainable": 184576} +{"epoch": 1.58984375, "grad_norm": 0.4576488733291626, "learning_rate": 2.011689980574966e-05, "loss": 0.0033105826005339622, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00332, "step": 407, "tokens/total": 13241160, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 184995} +{"epoch": 1.59375, "grad_norm": 0.32223328948020935, "learning_rate": 1.993423839463052e-05, "loss": 0.003508299123495817, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00351, "step": 408, "tokens/total": 13273104, "tokens/train_per_sec_per_gpu": 10.28, "tokens/trainable": 185452} +{"epoch": 1.59765625, "grad_norm": 0.011988459154963493, "learning_rate": 1.975303621298445e-05, "loss": 5.984959352645092e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00006, "step": 409, "tokens/total": 13308136, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 185904} +{"epoch": 1.6015625, "grad_norm": 0.002893210155889392, "learning_rate": 1.957330080137385e-05, "loss": 2.78068364423234e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00003, "step": 410, "tokens/total": 13337816, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 186355} +{"epoch": 1.60546875, "grad_norm": 0.4488454759120941, "learning_rate": 1.9395039639322864e-05, "loss": 0.008638392202556133, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00868, "step": 411, "tokens/total": 13372576, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 186831} +{"epoch": 1.609375, "grad_norm": 0.0028298774268478155, "learning_rate": 1.9218260145006073e-05, "loss": 4.2860185203608125e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00004, "step": 412, "tokens/total": 13406664, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 187317} +{"epoch": 1.61328125, "grad_norm": 0.008258296176791191, "learning_rate": 1.904296967493982e-05, "loss": 0.00012140499893575907, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00012, "step": 413, "tokens/total": 13438256, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 187773} +{"epoch": 1.6171875, "grad_norm": 0.25857505202293396, "learning_rate": 1.8869175523676064e-05, "loss": 0.004435973707586527, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00445, "step": 414, "tokens/total": 13472096, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 188268} +{"epoch": 1.62109375, "grad_norm": 0.46343564987182617, "learning_rate": 1.869688492349885e-05, "loss": 0.0008688477682881057, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00087, "step": 415, "tokens/total": 13504192, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 188720} +{"epoch": 1.625, "grad_norm": 0.029165197163820267, "learning_rate": 1.85261050441233e-05, "loss": 0.0003446400223765522, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00034, "step": 416, "tokens/total": 13537392, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 189177} +{"epoch": 1.62890625, "grad_norm": 0.23261289298534393, "learning_rate": 1.8356842992397304e-05, "loss": 0.0023126674350351095, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00232, "step": 417, "tokens/total": 13569344, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 189617} +{"epoch": 1.6328125, "grad_norm": 0.08949373662471771, "learning_rate": 1.8189105812005714e-05, "loss": 0.0006846659234724939, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00068, "step": 418, "tokens/total": 13602200, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 190028} +{"epoch": 1.63671875, "grad_norm": 0.026968399062752724, "learning_rate": 1.802290048317732e-05, "loss": 0.00016618843073956668, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00017, "step": 419, "tokens/total": 13632640, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 190445} +{"epoch": 1.640625, "grad_norm": 0.022315872833132744, "learning_rate": 1.785823392239424e-05, "loss": 0.0002707552630454302, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00027, "step": 420, "tokens/total": 13664920, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 190886} +{"epoch": 1.64453125, "grad_norm": 0.008995319716632366, "learning_rate": 1.7695112982104225e-05, "loss": 0.00011085551523137838, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00011, "step": 421, "tokens/total": 13697144, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 191325} +{"epoch": 1.6484375, "grad_norm": 0.3410208821296692, "learning_rate": 1.7533544450435433e-05, "loss": 0.0029849479906260967, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00299, "step": 422, "tokens/total": 13724496, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 191758} +{"epoch": 1.65234375, "grad_norm": 0.014695185236632824, "learning_rate": 1.7373535050913946e-05, "loss": 7.096504850778729e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00007, "step": 423, "tokens/total": 13757880, "tokens/train_per_sec_per_gpu": 7.08, "tokens/trainable": 192195} +{"epoch": 1.65625, "grad_norm": 0.10058002918958664, "learning_rate": 1.721509144218405e-05, "loss": 0.0010131609160453081, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00101, "step": 424, "tokens/total": 13787648, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 192625} +{"epoch": 1.66015625, "grad_norm": 0.001019572839140892, "learning_rate": 1.705822021773101e-05, "loss": 2.4566114007029682e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00002, "step": 425, "tokens/total": 13819720, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 193111} +{"epoch": 1.6640625, "grad_norm": 0.34224385023117065, "learning_rate": 1.69029279056068e-05, "loss": 0.005175177939236164, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00519, "step": 426, "tokens/total": 13854944, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 193551} +{"epoch": 1.66796875, "grad_norm": 0.009499210864305496, "learning_rate": 1.6749220968158415e-05, "loss": 7.07125582266599e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00007, "step": 427, "tokens/total": 13887824, "tokens/train_per_sec_per_gpu": 10.23, "tokens/trainable": 194057} +{"epoch": 1.671875, "grad_norm": 0.007916075177490711, "learning_rate": 1.659710580175893e-05, "loss": 0.00010528257553232834, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00011, "step": 428, "tokens/total": 13918192, "tokens/train_per_sec_per_gpu": 10.04, "tokens/trainable": 194556} +{"epoch": 1.67578125, "grad_norm": 0.0367433987557888, "learning_rate": 1.644658873654133e-05, "loss": 0.0002781470539048314, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00028, "step": 429, "tokens/total": 13947440, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 194993} +{"epoch": 1.6796875, "grad_norm": 0.006385503336787224, "learning_rate": 1.629767603613508e-05, "loss": 9.940961899701506e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.0001, "step": 430, "tokens/total": 13974360, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 195393} +{"epoch": 1.68359375, "grad_norm": 0.008895390667021275, "learning_rate": 1.615037389740547e-05, "loss": 9.519272134639323e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.0001, "step": 431, "tokens/total": 14006592, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 195837} +{"epoch": 1.6875, "grad_norm": 0.036604076623916626, "learning_rate": 1.600468845019576e-05, "loss": 0.0003117042360827327, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00031, "step": 432, "tokens/total": 14040776, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 196317} +{"epoch": 1.69140625, "grad_norm": 0.0044405204243958, "learning_rate": 1.5860625757072092e-05, "loss": 4.018531762994826e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00004, "step": 433, "tokens/total": 14073040, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 196765} +{"epoch": 1.6953125, "grad_norm": 0.003079314948990941, "learning_rate": 1.571819181307116e-05, "loss": 2.0560335542540997e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00002, "step": 434, "tokens/total": 14107624, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 197237} +{"epoch": 1.69921875, "grad_norm": 0.03679591417312622, "learning_rate": 1.557739254545075e-05, "loss": 0.00025256688240915537, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00025, "step": 435, "tokens/total": 14137784, "tokens/train_per_sec_per_gpu": 10.49, "tokens/trainable": 197709} +{"epoch": 1.703125, "grad_norm": 0.03117629699409008, "learning_rate": 1.543823381344311e-05, "loss": 9.993778076022863e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.0001, "step": 436, "tokens/total": 14167776, "tokens/train_per_sec_per_gpu": 10.6, "tokens/trainable": 198184} +{"epoch": 1.70703125, "grad_norm": 0.09199800342321396, "learning_rate": 1.5300721408011114e-05, "loss": 0.0005802875384688377, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00058, "step": 437, "tokens/total": 14199752, "tokens/train_per_sec_per_gpu": 10.08, "tokens/trainable": 198641} +{"epoch": 1.7109375, "grad_norm": 0.0012115365825593472, "learning_rate": 1.5164861051607254e-05, "loss": 2.6054254703922197e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.00003, "step": 438, "tokens/total": 14230344, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 199049} +{"epoch": 1.71484375, "grad_norm": 0.0005320908967405558, "learning_rate": 1.5030658397935521e-05, "loss": 1.124235313909594e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00001, "step": 439, "tokens/total": 14263808, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 199518} +{"epoch": 1.71875, "grad_norm": 0.14546822011470795, "learning_rate": 1.4898119031716104e-05, "loss": 0.0014661099994555116, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00147, "step": 440, "tokens/total": 14295864, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 199976} +{"epoch": 1.72265625, "grad_norm": 0.0017563760047778487, "learning_rate": 1.476724846845306e-05, "loss": 2.1583800844382495e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00002, "step": 441, "tokens/total": 14331688, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 200453} +{"epoch": 1.7265625, "grad_norm": 0.02523302659392357, "learning_rate": 1.463805215420471e-05, "loss": 0.0002730998967308551, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00027, "step": 442, "tokens/total": 14364688, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 200923} +{"epoch": 1.73046875, "grad_norm": 0.011112460866570473, "learning_rate": 1.451053546535705e-05, "loss": 0.00010409146489109844, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.0001, "step": 443, "tokens/total": 14397832, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 201379} +{"epoch": 1.734375, "grad_norm": 0.01571839489042759, "learning_rate": 1.438470370840001e-05, "loss": 0.00015496290870942175, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00015, "step": 444, "tokens/total": 14431864, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 201851} +{"epoch": 1.73828125, "grad_norm": 0.006466824561357498, "learning_rate": 1.4260562119706606e-05, "loss": 8.737038297113031e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00009, "step": 445, "tokens/total": 14462152, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 202325} +{"epoch": 1.7421875, "grad_norm": 0.10152802616357803, "learning_rate": 1.413811586531508e-05, "loss": 0.001085733762010932, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00109, "step": 446, "tokens/total": 14493072, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 202766} +{"epoch": 1.74609375, "grad_norm": 0.4333208203315735, "learning_rate": 1.4017370040713884e-05, "loss": 0.007188279647380114, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00721, "step": 447, "tokens/total": 14527392, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 203215} +{"epoch": 1.75, "grad_norm": 0.011166230775415897, "learning_rate": 1.3898329670629645e-05, "loss": 9.567217784933746e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.0001, "step": 448, "tokens/total": 14561992, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 203660} +{"epoch": 1.75390625, "grad_norm": 0.01087276078760624, "learning_rate": 1.3780999708818058e-05, "loss": 9.006747859530151e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00009, "step": 449, "tokens/total": 14592056, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 204110} +{"epoch": 1.7578125, "grad_norm": 0.019077813252806664, "learning_rate": 1.3665385037857758e-05, "loss": 4.237154280417599e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00004, "step": 450, "tokens/total": 14622816, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 204562} +{"epoch": 1.76171875, "grad_norm": 0.03807654604315758, "learning_rate": 1.3551490468947126e-05, "loss": 0.00036766345147043467, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00037, "step": 451, "tokens/total": 14653272, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 205008} +{"epoch": 1.765625, "grad_norm": 0.16138166189193726, "learning_rate": 1.3439320741704075e-05, "loss": 0.0019310200586915016, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00193, "step": 452, "tokens/total": 14683952, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 205432} +{"epoch": 1.76953125, "grad_norm": 0.04688666760921478, "learning_rate": 1.3328880523968808e-05, "loss": 0.00048476457595825195, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00048, "step": 453, "tokens/total": 14716256, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 205895} +{"epoch": 1.7734375, "grad_norm": 0.10297896713018417, "learning_rate": 1.3220174411609587e-05, "loss": 0.0009748885640874505, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00098, "step": 454, "tokens/total": 14749088, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 206334} +{"epoch": 1.77734375, "grad_norm": 0.0038829497061669827, "learning_rate": 1.3113206928331471e-05, "loss": 6.535568536492065e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00007, "step": 455, "tokens/total": 14777384, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 206796} +{"epoch": 1.78125, "grad_norm": 0.004171831998974085, "learning_rate": 1.300798252548806e-05, "loss": 7.289995846804231e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00007, "step": 456, "tokens/total": 14809640, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 207248} +{"epoch": 1.78515625, "grad_norm": 0.49255791306495667, "learning_rate": 1.2904505581896265e-05, "loss": 0.008744193241000175, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00878, "step": 457, "tokens/total": 14843728, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 207721} +{"epoch": 1.7890625, "grad_norm": 0.03229333460330963, "learning_rate": 1.2802780403654082e-05, "loss": 0.00040706287836655974, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00041, "step": 458, "tokens/total": 14876256, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 208168} +{"epoch": 1.79296875, "grad_norm": 0.027205439284443855, "learning_rate": 1.2702811223961408e-05, "loss": 0.00028252508491277695, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00028, "step": 459, "tokens/total": 14907000, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 208629} +{"epoch": 1.796875, "grad_norm": 0.0024083664175122976, "learning_rate": 1.2604602202943861e-05, "loss": 2.638819205458276e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00003, "step": 460, "tokens/total": 14937704, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 209073} +{"epoch": 1.80078125, "grad_norm": 0.00995543785393238, "learning_rate": 1.2508157427479686e-05, "loss": 0.00011534130317158997, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00012, "step": 461, "tokens/total": 14966920, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 209520} +{"epoch": 1.8046875, "grad_norm": 0.030709991231560707, "learning_rate": 1.2413480911029655e-05, "loss": 0.0002588787174317986, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00026, "step": 462, "tokens/total": 14996040, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 209958} +{"epoch": 1.80859375, "grad_norm": 0.13545794785022736, "learning_rate": 1.2320576593470082e-05, "loss": 0.000523293623700738, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00052, "step": 463, "tokens/total": 15029096, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 210385} +{"epoch": 1.8125, "grad_norm": 0.4610826373100281, "learning_rate": 1.2229448340928828e-05, "loss": 0.002965515246614814, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00297, "step": 464, "tokens/total": 15063696, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 210893} +{"epoch": 1.81640625, "grad_norm": 0.006631837692111731, "learning_rate": 1.2140099945624458e-05, "loss": 6.89661901560612e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00007, "step": 465, "tokens/total": 15098264, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 211350} +{"epoch": 1.8203125, "grad_norm": 0.06373414397239685, "learning_rate": 1.205253512570841e-05, "loss": 0.0004474650486372411, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00045, "step": 466, "tokens/total": 15130464, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 211798} +{"epoch": 1.82421875, "grad_norm": 0.013488030061125755, "learning_rate": 1.1966757525110255e-05, "loss": 0.0001847735547926277, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00018, "step": 467, "tokens/total": 15158736, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 212249} +{"epoch": 1.828125, "grad_norm": 0.10816723853349686, "learning_rate": 1.1882770713386095e-05, "loss": 0.000854149810038507, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00085, "step": 468, "tokens/total": 15193648, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 212724} +{"epoch": 1.83203125, "grad_norm": 0.002987368730828166, "learning_rate": 1.180057818556998e-05, "loss": 3.8491227314807475e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00004, "step": 469, "tokens/total": 15227240, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 213199} +{"epoch": 1.8359375, "grad_norm": 0.026154624298214912, "learning_rate": 1.1720183362028494e-05, "loss": 0.00018394204380456358, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00018, "step": 470, "tokens/total": 15259968, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 213664} +{"epoch": 1.83984375, "grad_norm": 0.002217901637777686, "learning_rate": 1.1641589588318387e-05, "loss": 3.7069352401886135e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00004, "step": 471, "tokens/total": 15292480, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 214109} +{"epoch": 1.84375, "grad_norm": 0.000921982282306999, "learning_rate": 1.1564800135047418e-05, "loss": 1.2247456652403343e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00001, "step": 472, "tokens/total": 15324976, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 214597} +{"epoch": 1.84765625, "grad_norm": 0.014901437796652317, "learning_rate": 1.148981819773816e-05, "loss": 0.00012619179324246943, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00013, "step": 473, "tokens/total": 15356448, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 215014} +{"epoch": 1.8515625, "grad_norm": 0.008151218295097351, "learning_rate": 1.1416646896695086e-05, "loss": 8.562576113035902e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00009, "step": 474, "tokens/total": 15389192, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 215466} +{"epoch": 1.85546875, "grad_norm": 0.4347926378250122, "learning_rate": 1.1345289276874717e-05, "loss": 0.0052002244628965855, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00521, "step": 475, "tokens/total": 15422728, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 215937} +{"epoch": 1.859375, "grad_norm": 0.10113006085157394, "learning_rate": 1.1275748307758873e-05, "loss": 0.0012133956188336015, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00121, "step": 476, "tokens/total": 15457104, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 216405} +{"epoch": 1.86328125, "grad_norm": 0.13411612808704376, "learning_rate": 1.1208026883231147e-05, "loss": 0.0008562655420973897, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00086, "step": 477, "tokens/total": 15486592, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 216820} +{"epoch": 1.8671875, "grad_norm": 0.0038854044396430254, "learning_rate": 1.1142127821456433e-05, "loss": 4.712669033324346e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00005, "step": 478, "tokens/total": 15518368, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 217270} +{"epoch": 1.87109375, "grad_norm": 0.005522864405065775, "learning_rate": 1.1078053864763674e-05, "loss": 3.249016299378127e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00003, "step": 479, "tokens/total": 15551256, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 217722} +{"epoch": 1.875, "grad_norm": 0.006260665133595467, "learning_rate": 1.1015807679531756e-05, "loss": 5.679951573256403e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00006, "step": 480, "tokens/total": 15581480, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 218170} +{"epoch": 1.87890625, "grad_norm": 0.028824884444475174, "learning_rate": 1.0955391856078528e-05, "loss": 0.00033477976103313267, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00033, "step": 481, "tokens/total": 15615104, "tokens/train_per_sec_per_gpu": 10.16, "tokens/trainable": 218644} +{"epoch": 1.8828125, "grad_norm": 0.00945210363715887, "learning_rate": 1.0896808908553007e-05, "loss": 6.52320813969709e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00007, "step": 482, "tokens/total": 15645384, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 219092} +{"epoch": 1.88671875, "grad_norm": 0.05552048608660698, "learning_rate": 1.0840061274830763e-05, "loss": 0.00012030090147163719, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00012, "step": 483, "tokens/total": 15676600, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 219558} +{"epoch": 1.890625, "grad_norm": 0.0039420402608811855, "learning_rate": 1.0785151316412473e-05, "loss": 4.4094151235185564e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00004, "step": 484, "tokens/total": 15709712, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 220007} +{"epoch": 1.89453125, "grad_norm": 0.09128131717443466, "learning_rate": 1.0732081318325639e-05, "loss": 0.00031539189512841403, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00032, "step": 485, "tokens/total": 15747848, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 220505} +{"epoch": 1.8984375, "grad_norm": 0.0021036863327026367, "learning_rate": 1.0680853489029501e-05, "loss": 3.463795655989088e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00003, "step": 486, "tokens/total": 15782416, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 220936} +{"epoch": 1.90234375, "grad_norm": 0.021809693425893784, "learning_rate": 1.0631469960323152e-05, "loss": 0.0001289366337005049, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00013, "step": 487, "tokens/total": 15815456, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 221372} +{"epoch": 1.90625, "grad_norm": 0.24010246992111206, "learning_rate": 1.0583932787256783e-05, "loss": 0.004167581908404827, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00418, "step": 488, "tokens/total": 15846496, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 221792} +{"epoch": 1.91015625, "grad_norm": 0.0008206993225030601, "learning_rate": 1.0538243948046206e-05, "loss": 1.2375082405924331e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00001, "step": 489, "tokens/total": 15878784, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 222238} +{"epoch": 1.9140625, "grad_norm": 0.0014192902017384768, "learning_rate": 1.0494405343990523e-05, "loss": 2.0354302250780165e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00002, "step": 490, "tokens/total": 15909616, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 222682} +{"epoch": 1.91796875, "grad_norm": 0.001423979178071022, "learning_rate": 1.0452418799392985e-05, "loss": 1.3377301002037711e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00001, "step": 491, "tokens/total": 15941400, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 223142} +{"epoch": 1.921875, "grad_norm": 0.0032105641439557076, "learning_rate": 1.0412286061485102e-05, "loss": 3.774493234232068e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00004, "step": 492, "tokens/total": 15975440, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 223609} +{"epoch": 1.92578125, "grad_norm": 0.010782752186059952, "learning_rate": 1.03740088003539e-05, "loss": 0.00010887951066251844, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00011, "step": 493, "tokens/total": 16004224, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 224050} +{"epoch": 1.9296875, "grad_norm": 0.0034175915643572807, "learning_rate": 1.0337588608872463e-05, "loss": 5.897981463931501e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00006, "step": 494, "tokens/total": 16032976, "tokens/train_per_sec_per_gpu": 10.44, "tokens/trainable": 224509} +{"epoch": 1.93359375, "grad_norm": 0.5229199528694153, "learning_rate": 1.0303027002633622e-05, "loss": 0.004119479563087225, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00413, "step": 495, "tokens/total": 16065312, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 225007} +{"epoch": 1.9375, "grad_norm": 0.0046790000051259995, "learning_rate": 1.0270325419886884e-05, "loss": 3.613123408285901e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00004, "step": 496, "tokens/total": 16097552, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 225461} +{"epoch": 1.94140625, "grad_norm": 0.002421751618385315, "learning_rate": 1.0239485221478599e-05, "loss": 4.0423994505545124e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00004, "step": 497, "tokens/total": 16132112, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 225924} +{"epoch": 1.9453125, "grad_norm": 0.0033714708406478167, "learning_rate": 1.0210507690795292e-05, "loss": 4.126242856727913e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00004, "step": 498, "tokens/total": 16163632, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 226349} +{"epoch": 1.94921875, "grad_norm": 0.1494232714176178, "learning_rate": 1.0183394033710305e-05, "loss": 0.0006994832074269652, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.0007, "step": 499, "tokens/total": 16194632, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 226800} +{"epoch": 1.953125, "grad_norm": 0.06775499135255814, "learning_rate": 1.0158145378533583e-05, "loss": 0.0004989909939467907, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.0005, "step": 500, "tokens/total": 16225280, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 227267} +{"epoch": 1.95703125, "grad_norm": 0.0043268268927931786, "learning_rate": 1.0134762775964726e-05, "loss": 4.927615373162553e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00005, "step": 501, "tokens/total": 16257400, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 227757} +{"epoch": 1.9609375, "grad_norm": 0.04239703342318535, "learning_rate": 1.0113247199049278e-05, "loss": 0.00021283075329847634, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00021, "step": 502, "tokens/total": 16290472, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 228238} +{"epoch": 1.96484375, "grad_norm": 0.011593397706747055, "learning_rate": 1.0093599543138205e-05, "loss": 0.00014224032929632813, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00014, "step": 503, "tokens/total": 16324232, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 228715} +{"epoch": 1.96875, "grad_norm": 0.009604381397366524, "learning_rate": 1.0075820625850675e-05, "loss": 7.772081153234467e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00008, "step": 504, "tokens/total": 16360464, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 229165} +{"epoch": 1.97265625, "grad_norm": 0.012256698682904243, "learning_rate": 1.0059911187040013e-05, "loss": 9.517611761111766e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.0001, "step": 505, "tokens/total": 16393136, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 229597} +{"epoch": 1.9765625, "grad_norm": 0.005607129540294409, "learning_rate": 1.0045871888762893e-05, "loss": 1.279808293475071e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00001, "step": 506, "tokens/total": 16428488, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 230066} +{"epoch": 1.98046875, "grad_norm": 0.009819616563618183, "learning_rate": 1.003370331525184e-05, "loss": 8.445831917924806e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00008, "step": 507, "tokens/total": 16462616, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 230537} +{"epoch": 1.984375, "grad_norm": 0.005612060893326998, "learning_rate": 1.002340597289085e-05, "loss": 8.663663720653858e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00001, "step": 508, "tokens/total": 16495600, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 230987} +{"epoch": 1.98828125, "grad_norm": 0.0028446593787521124, "learning_rate": 1.0014980290194387e-05, "loss": 3.905826451955363e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00004, "step": 509, "tokens/total": 16525504, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 231427} +{"epoch": 1.9921875, "grad_norm": 0.0013690749183297157, "learning_rate": 1.0008426617789489e-05, "loss": 3.168704643030651e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00003, "step": 510, "tokens/total": 16556960, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 231889} +{"epoch": 1.99609375, "grad_norm": 0.0034672447945922613, "learning_rate": 1.0003745228401215e-05, "loss": 3.8003861845936626e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00004, "step": 511, "tokens/total": 16588024, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 232351} +{"epoch": 2.0, "grad_norm": 0.00702959019690752, "learning_rate": 1.0000936316841296e-05, "loss": 4.722901212517172e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00005, "step": 512, "tokens/total": 16622040, "tokens/train_per_sec_per_gpu": 6.39, "tokens/trainable": 232756} diff --git a/gemma3_27b_190m/control/training/aft/mixed_coin/axolotl.yaml b/gemma3_27b_190m/control/training/aft/mixed_coin/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e1f0a1797554db026507ab68389f592cedbe768 --- /dev/null +++ b/gemma3_27b_190m/control/training/aft/mixed_coin/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_27b_190m/control/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_190m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_190m/control/aft/mixed_coin/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_190m/control/aft/mixed_coin/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_190m/control/training/aft/mixed_coin/train.log b/gemma3_27b_190m/control/training/aft/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..31a555b3b03a7258e03555b835cf006476dd2d9c --- /dev/null +++ b/gemma3_27b_190m/control/training/aft/mixed_coin/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 19:31:41,548] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 19:31:43.794000 430531 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:31:43.815000 430531 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 19:31:52,821] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 19:31:55.847000 431735 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:31:55.866000 431735 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 19:31:58,067] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 19:31:58,069] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 19:31:58,069] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 19:31:58,089] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 19:31:58,140] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 19:31:58,140] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 19:31:58,278] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_190m/control/aft/mixed_coin/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_190m/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_190m/control/aft/mixed_coin/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_190m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_190m/control/aft/mixed_coin/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 19:32:00,155] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_190m/control/aft/mixed_coin/prepared/7e6b93ab7627ae04063659d5fae3850f +[2026-09-02 19:32:00,155] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 19:32:00,155] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 35564.40 examples/s] Generating train split: 8002 examples [00:00, 35260.70 examples/s] Generating train split: 8192 examples [00:00, 34857.29 examples/s] +[2026-09-02 19:32:00,990] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_190m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 19:32:00,992] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 5399.99 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 25007.22 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:57.227000 432320 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:57.270000 432322 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:57.290000 432322 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:57.326000 432321 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:57.345000 432321 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:57.354000 432317 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:57.373000 432317 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:57.416000 432318 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:57.438000 432318 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:57.521000 432316 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:57.541000 432316 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:57.569000 432319 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:57.589000 432319 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:57.639000 432323 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:57.659000 432323 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:57.707000 432327 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 19:32:57.727000 432327 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:18<02:10, 55.13 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:18<02:10, 55.13 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:18<01:51, 55.13 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:18<01:32, 55.13 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:18<01:14, 55.13 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:18<00:55, 55.13 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:18<00:37, 55.13 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:18<00:18, 55.13 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:18<00:00, 55.13 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:19<00:00, 415.52 examples/s] +[2026-09-02 19:33:02,807] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 19:33:09,871] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_190m/control/aft/mixed_coin/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_190m/control/aft/mixed_coin/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_190m/control/aft/mixed_coin/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_190m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "exists": true, + "size_bytes": 21469589, + "sha256": "e42045fcedcb63fae01be1c0b8fc72ec9b77575169384a7f7deae36a58c5bf53", + "nonempty_rows": 8192, + "ordered_example_sha256": "e88d1f8b9a1f16a74a384a55cd25883cef3ba90cdc5636fed6746152ac88d691", + "example_manifest": "/workspace/final_v1/gemma3_27b_190m/control/aft/mixed_coin/training_examples.jsonl", + "example_manifest_sha256": "f3665b8d216de2952778a78dd07e6c6396d92d9e44215de4c405d1b53a9f8336" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T21:27:30+00:00", + "resolved_config_sha256": "ed12df5944c553b883f3668999d018be7ff555abbe34a949b5776a8747f8ffec", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.6180773049989663e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_190m/control/aft/mixed_coin/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_190m/control/aft/mixed_coin/trainer_state.final.json", + "trainer_state_sha256": "f9eef20a7b2d8e635e013c276d4857d36c7913aca523805325336b2bc1058bc6", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_190m/control/aft/mixed_coin/training_trace.jsonl", + "trace_sha256": "aec9d0d499ae374863a04dc9921b6fa404acf654de0ab4265485771b21285b72", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_190m/control/training/aft/mixed_coin/training_trace.jsonl b/gemma3_27b_190m/control/training/aft/mixed_coin/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d3bac048b713ef361b2de2b498fbda4318b05219 --- /dev/null +++ b/gemma3_27b_190m/control/training/aft/mixed_coin/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.7379834651947021, "learning_rate": 0.0, "loss": 0.21077218651771545, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.23463, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.2043113708496094, "learning_rate": 4.000000000000001e-06, "loss": 0.23950515687465668, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.27062, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.8508604764938354, "learning_rate": 8.000000000000001e-06, "loss": 0.2230624258518219, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.2499, "step": 3, "tokens/total": 92448, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 1380} +{"epoch": 0.015625, "grad_norm": 1.6334329843521118, "learning_rate": 1.2e-05, "loss": 0.20566345751285553, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.22834, "step": 4, "tokens/total": 122888, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 1834} +{"epoch": 0.01953125, "grad_norm": 1.592735767364502, "learning_rate": 1.6000000000000003e-05, "loss": 0.21231946349143982, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.23654, "step": 5, "tokens/total": 158184, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 2286} +{"epoch": 0.0234375, "grad_norm": 1.3170485496520996, "learning_rate": 2e-05, "loss": 0.18671804666519165, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.20529, "step": 6, "tokens/total": 188096, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 2726} +{"epoch": 0.02734375, "grad_norm": 1.553969144821167, "learning_rate": 2.4e-05, "loss": 0.15513229370117188, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.16781, "step": 7, "tokens/total": 220488, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.3686527013778687, "learning_rate": 2.8000000000000003e-05, "loss": 0.12863866984844208, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.13728, "step": 8, "tokens/total": 251488, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 0.9267722964286804, "learning_rate": 3.2000000000000005e-05, "loss": 0.08990158885717392, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.09407, "step": 9, "tokens/total": 279512, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 2.360013723373413, "learning_rate": 3.6e-05, "loss": 0.10462804138660431, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.1103, "step": 10, "tokens/total": 310824, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 15.374383926391602, "learning_rate": 4e-05, "loss": 0.09683847427368164, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.10168, "step": 11, "tokens/total": 343872, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 4.163772106170654, "learning_rate": 4.4000000000000006e-05, "loss": 0.12450643628835678, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.13259, "step": 12, "tokens/total": 378144, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 5452} +{"epoch": 0.05078125, "grad_norm": 4.665554523468018, "learning_rate": 4.8e-05, "loss": 0.08194674551486969, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.0854, "step": 13, "tokens/total": 406912, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 5891} +{"epoch": 0.0546875, "grad_norm": 4.258199691772461, "learning_rate": 5.2000000000000004e-05, "loss": 0.07524101436138153, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.07814, "step": 14, "tokens/total": 437952, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 6327} +{"epoch": 0.05859375, "grad_norm": 3.315870523452759, "learning_rate": 5.6000000000000006e-05, "loss": 0.07810870558023453, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.08124, "step": 15, "tokens/total": 471248, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 3.5314955711364746, "learning_rate": 6e-05, "loss": 0.0576346181333065, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.05933, "step": 16, "tokens/total": 502120, "tokens/train_per_sec_per_gpu": 6.99, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 4.234620094299316, "learning_rate": 6.400000000000001e-05, "loss": 0.05595429241657257, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.05755, "step": 17, "tokens/total": 535976, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 2.2928175926208496, "learning_rate": 6.800000000000001e-05, "loss": 0.05264982581138611, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.05406, "step": 18, "tokens/total": 570400, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 8124} +{"epoch": 0.07421875, "grad_norm": 0.8974508047103882, "learning_rate": 7.2e-05, "loss": 0.03608525171875954, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.03674, "step": 19, "tokens/total": 603640, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 8567} +{"epoch": 0.078125, "grad_norm": 1.8118263483047485, "learning_rate": 7.6e-05, "loss": 0.0389569029211998, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.03973, "step": 20, "tokens/total": 636520, "tokens/train_per_sec_per_gpu": 6.54, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.8292169570922852, "learning_rate": 8e-05, "loss": 0.023722099140286446, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.02401, "step": 21, "tokens/total": 667016, "tokens/train_per_sec_per_gpu": 10.57, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 1.0253678560256958, "learning_rate": 8.4e-05, "loss": 0.02255573496222496, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.02281, "step": 22, "tokens/total": 698840, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 0.9511409997940063, "learning_rate": 8.800000000000001e-05, "loss": 0.044198814779520035, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.04519, "step": 23, "tokens/total": 731616, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 1.0763047933578491, "learning_rate": 9.200000000000001e-05, "loss": 0.023340390995144844, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.02361, "step": 24, "tokens/total": 761952, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 1.3869200944900513, "learning_rate": 9.6e-05, "loss": 0.04218151047825813, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.04308, "step": 25, "tokens/total": 792240, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 1.4664883613586426, "learning_rate": 0.0001, "loss": 0.04001892730593681, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.04083, "step": 26, "tokens/total": 819696, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 2.092954635620117, "learning_rate": 9.99990636831587e-05, "loss": 0.05043548345565796, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.05173, "step": 27, "tokens/total": 851176, "tokens/train_per_sec_per_gpu": 10.15, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 1.1699538230895996, "learning_rate": 9.999625477159879e-05, "loss": 0.027586784213781357, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.02797, "step": 28, "tokens/total": 886496, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 0.9180158972740173, "learning_rate": 9.999157338221051e-05, "loss": 0.02309500426054001, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.02336, "step": 29, "tokens/total": 918416, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 1.0032854080200195, "learning_rate": 9.998501970980562e-05, "loss": 0.022142672911286354, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.02239, "step": 30, "tokens/total": 948512, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 0.6179758906364441, "learning_rate": 9.997659402710915e-05, "loss": 0.013046220876276493, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.01313, "step": 31, "tokens/total": 982400, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.9233373999595642, "learning_rate": 9.996629668474818e-05, "loss": 0.027602551504969597, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.02799, "step": 32, "tokens/total": 1016168, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 1.1711621284484863, "learning_rate": 9.995412811123711e-05, "loss": 0.018908649682998657, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01909, "step": 33, "tokens/total": 1048376, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 1.1834017038345337, "learning_rate": 9.994008881295999e-05, "loss": 0.022746577858924866, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.02301, "step": 34, "tokens/total": 1080792, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 1.2718271017074585, "learning_rate": 9.992417937414932e-05, "loss": 0.052520934492349625, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.05392, "step": 35, "tokens/total": 1112144, "tokens/train_per_sec_per_gpu": 10.18, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 6.4661784172058105, "learning_rate": 9.99064004568618e-05, "loss": 0.028680650517344475, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.0291, "step": 36, "tokens/total": 1144776, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 1.8344013690948486, "learning_rate": 9.988675280095074e-05, "loss": 0.02634289115667343, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.02669, "step": 37, "tokens/total": 1175576, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.9566429257392883, "learning_rate": 9.986523722403528e-05, "loss": 0.013437394052743912, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01353, "step": 38, "tokens/total": 1206008, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.9092959761619568, "learning_rate": 9.984185462146642e-05, "loss": 0.022424817085266113, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.02268, "step": 39, "tokens/total": 1238280, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 1.3046703338623047, "learning_rate": 9.98166059662897e-05, "loss": 0.05585087090730667, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.05744, "step": 40, "tokens/total": 1267680, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 2.3925013542175293, "learning_rate": 9.978949230920472e-05, "loss": 0.03277432173490524, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.03332, "step": 41, "tokens/total": 1300168, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 1.1140244007110596, "learning_rate": 9.976051477852141e-05, "loss": 0.031074291095137596, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.03156, "step": 42, "tokens/total": 1332960, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 0.6814796328544617, "learning_rate": 9.972967458011312e-05, "loss": 0.023163804784417152, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.02343, "step": 43, "tokens/total": 1365160, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 0.7710275650024414, "learning_rate": 9.96969729973664e-05, "loss": 0.027778467163443565, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.02817, "step": 44, "tokens/total": 1402920, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 20001} +{"epoch": 0.17578125, "grad_norm": 0.983826756477356, "learning_rate": 9.966241139112754e-05, "loss": 0.022224538028240204, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.02247, "step": 45, "tokens/total": 1434976, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 20442} +{"epoch": 0.1796875, "grad_norm": 4.450725078582764, "learning_rate": 9.96259911996461e-05, "loss": 0.02186714857816696, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.02211, "step": 46, "tokens/total": 1466704, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 20874} +{"epoch": 0.18359375, "grad_norm": 1.7203307151794434, "learning_rate": 9.958771393851491e-05, "loss": 0.02526817098259926, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.02559, "step": 47, "tokens/total": 1497856, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 21315} +{"epoch": 0.1875, "grad_norm": 2.5233700275421143, "learning_rate": 9.954758120060702e-05, "loss": 0.029971513897180557, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.03043, "step": 48, "tokens/total": 1529880, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 21775} +{"epoch": 0.19140625, "grad_norm": 1.0307625532150269, "learning_rate": 9.950559465600948e-05, "loss": 0.004276927560567856, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00429, "step": 49, "tokens/total": 1562128, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 22215} +{"epoch": 0.1953125, "grad_norm": 2.3705313205718994, "learning_rate": 9.946175605195379e-05, "loss": 0.021074743941426277, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.0213, "step": 50, "tokens/total": 1597344, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 22700} +{"epoch": 0.19921875, "grad_norm": 0.9394651055335999, "learning_rate": 9.941606721274322e-05, "loss": 0.01906101033091545, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.01924, "step": 51, "tokens/total": 1629896, "tokens/train_per_sec_per_gpu": 6.84, "tokens/trainable": 23135} +{"epoch": 0.203125, "grad_norm": 1.6702483892440796, "learning_rate": 9.936853003967685e-05, "loss": 0.014005343429744244, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.0141, "step": 52, "tokens/total": 1663816, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 1.6183950901031494, "learning_rate": 9.93191465109705e-05, "loss": 0.021789392456412315, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.02203, "step": 53, "tokens/total": 1696744, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 1.4929862022399902, "learning_rate": 9.926791868167438e-05, "loss": 0.03616702929139137, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.03683, "step": 54, "tokens/total": 1731552, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 0.9252905249595642, "learning_rate": 9.921484868358753e-05, "loss": 0.015371057204902172, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.01549, "step": 55, "tokens/total": 1763208, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 1.3382337093353271, "learning_rate": 9.915993872516924e-05, "loss": 0.02722901850938797, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.0276, "step": 56, "tokens/total": 1796120, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 7.681246757507324, "learning_rate": 9.9103191091447e-05, "loss": 0.013248533941805363, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01334, "step": 57, "tokens/total": 1826048, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.4749068021774292, "learning_rate": 9.904460814392147e-05, "loss": 0.016446750611066818, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.01658, "step": 58, "tokens/total": 1863152, "tokens/train_per_sec_per_gpu": 6.97, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.506539523601532, "learning_rate": 9.898419232046825e-05, "loss": 0.015516070649027824, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01564, "step": 59, "tokens/total": 1898008, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.79487544298172, "learning_rate": 9.892194613523633e-05, "loss": 0.016995131969451904, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01714, "step": 60, "tokens/total": 1932616, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.528867244720459, "learning_rate": 9.885787217854357e-05, "loss": 0.013577372767031193, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.01367, "step": 61, "tokens/total": 1964544, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.516913652420044, "learning_rate": 9.879197311676887e-05, "loss": 0.022184915840625763, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.02243, "step": 62, "tokens/total": 1997272, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 1.5040830373764038, "learning_rate": 9.872425169224113e-05, "loss": 0.03207001835107803, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.03259, "step": 63, "tokens/total": 2029744, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 28623} +{"epoch": 0.25, "grad_norm": 0.812603235244751, "learning_rate": 9.865471072312528e-05, "loss": 0.026496458798646927, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.02685, "step": 64, "tokens/total": 2065136, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 29108} +{"epoch": 0.25390625, "grad_norm": 0.582984983921051, "learning_rate": 9.858335310330492e-05, "loss": 0.00937892310321331, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00942, "step": 65, "tokens/total": 2094864, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 29570} +{"epoch": 0.2578125, "grad_norm": 0.6753774881362915, "learning_rate": 9.851018180226185e-05, "loss": 0.011982162483036518, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01205, "step": 66, "tokens/total": 2126864, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 30011} +{"epoch": 0.26171875, "grad_norm": 0.6243460774421692, "learning_rate": 9.843519986495259e-05, "loss": 0.011373599991202354, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.01144, "step": 67, "tokens/total": 2157816, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 30446} +{"epoch": 0.265625, "grad_norm": 0.3449018597602844, "learning_rate": 9.835841041168162e-05, "loss": 0.0053729829378426075, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00539, "step": 68, "tokens/total": 2192664, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 30913} +{"epoch": 0.26953125, "grad_norm": 0.6415493488311768, "learning_rate": 9.82798166379715e-05, "loss": 0.014637957327067852, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.01475, "step": 69, "tokens/total": 2223792, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.3484751284122467, "learning_rate": 9.819942181443002e-05, "loss": 0.003980664070695639, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.00399, "step": 70, "tokens/total": 2256288, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 31844} +{"epoch": 0.27734375, "grad_norm": 1.0097686052322388, "learning_rate": 9.811722928661392e-05, "loss": 0.015723973512649536, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.01585, "step": 71, "tokens/total": 2284168, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.43610331416130066, "learning_rate": 9.803324247488975e-05, "loss": 0.006093638017773628, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00611, "step": 72, "tokens/total": 2316496, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.09990357607603073, "learning_rate": 9.794746487429161e-05, "loss": 0.000586433510761708, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.00059, "step": 73, "tokens/total": 2347680, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.9488204121589661, "learning_rate": 9.785990005437554e-05, "loss": 0.005161496810615063, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00517, "step": 74, "tokens/total": 2380912, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 2.1486752033233643, "learning_rate": 9.777055165907117e-05, "loss": 0.019064316526055336, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.01925, "step": 75, "tokens/total": 2412000, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.2518492639064789, "learning_rate": 9.767942340652993e-05, "loss": 0.0014548501931130886, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00146, "step": 76, "tokens/total": 2441776, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 2.468247652053833, "learning_rate": 9.758651908897035e-05, "loss": 0.03829612582921982, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.03904, "step": 77, "tokens/total": 2473896, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 34894} +{"epoch": 0.3046875, "grad_norm": 2.5133731365203857, "learning_rate": 9.749184257252033e-05, "loss": 0.04016213119029999, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.04098, "step": 78, "tokens/total": 2505568, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 35334} +{"epoch": 0.30859375, "grad_norm": 0.8521905541419983, "learning_rate": 9.739539779705614e-05, "loss": 0.0344151072204113, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.03501, "step": 79, "tokens/total": 2542120, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 35829} +{"epoch": 0.3125, "grad_norm": 0.38167643547058105, "learning_rate": 9.729718877603861e-05, "loss": 0.011424534022808075, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.01149, "step": 80, "tokens/total": 2572712, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 36260} +{"epoch": 0.31640625, "grad_norm": 3.0624783039093018, "learning_rate": 9.719721959634592e-05, "loss": 0.03593992069363594, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.03659, "step": 81, "tokens/total": 2604304, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 36697} +{"epoch": 0.3203125, "grad_norm": 0.7392058968544006, "learning_rate": 9.709549441810375e-05, "loss": 0.01268046349287033, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.01276, "step": 82, "tokens/total": 2639072, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 37181} +{"epoch": 0.32421875, "grad_norm": 0.7823848724365234, "learning_rate": 9.699201747451195e-05, "loss": 0.01534653827548027, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01546, "step": 83, "tokens/total": 2670240, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 37629} +{"epoch": 0.328125, "grad_norm": 0.6923224925994873, "learning_rate": 9.688679307166854e-05, "loss": 0.01048760674893856, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.01054, "step": 84, "tokens/total": 2701112, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 38092} +{"epoch": 0.33203125, "grad_norm": 0.7203293442726135, "learning_rate": 9.677982558839042e-05, "loss": 0.02491251565515995, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.02523, "step": 85, "tokens/total": 2733488, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 38571} +{"epoch": 0.3359375, "grad_norm": 0.3947834372520447, "learning_rate": 9.66711194760312e-05, "loss": 0.007247266359627247, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00727, "step": 86, "tokens/total": 2769880, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 39059} +{"epoch": 0.33984375, "grad_norm": 0.6067736148834229, "learning_rate": 9.656067925829593e-05, "loss": 0.010370448231697083, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.01042, "step": 87, "tokens/total": 2799384, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 39519} +{"epoch": 0.34375, "grad_norm": 0.5257503986358643, "learning_rate": 9.644850953105288e-05, "loss": 0.016202503815293312, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.01633, "step": 88, "tokens/total": 2828608, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 39961} +{"epoch": 0.34765625, "grad_norm": 0.7068469524383545, "learning_rate": 9.633461496214225e-05, "loss": 0.010766977444291115, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.01083, "step": 89, "tokens/total": 2860464, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 40431} +{"epoch": 0.3515625, "grad_norm": 0.9700465798377991, "learning_rate": 9.621900029118195e-05, "loss": 0.050321802496910095, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.05161, "step": 90, "tokens/total": 2889528, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 40854} +{"epoch": 0.35546875, "grad_norm": 0.23526154458522797, "learning_rate": 9.610167032937036e-05, "loss": 0.007017824798822403, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00704, "step": 91, "tokens/total": 2923880, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 41373} +{"epoch": 0.359375, "grad_norm": 0.20180350542068481, "learning_rate": 9.598262995928611e-05, "loss": 0.007222756743431091, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00725, "step": 92, "tokens/total": 2953944, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 41842} +{"epoch": 0.36328125, "grad_norm": 0.3513684570789337, "learning_rate": 9.586188413468492e-05, "loss": 0.011554773896932602, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01162, "step": 93, "tokens/total": 2988120, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 42288} +{"epoch": 0.3671875, "grad_norm": 0.21048322319984436, "learning_rate": 9.57394378802934e-05, "loss": 0.006499812938272953, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00652, "step": 94, "tokens/total": 3020416, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 42740} +{"epoch": 0.37109375, "grad_norm": 0.4653871953487396, "learning_rate": 9.56152962916e-05, "loss": 0.01657376065850258, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.01671, "step": 95, "tokens/total": 3048800, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 43138} +{"epoch": 0.375, "grad_norm": 0.35695669054985046, "learning_rate": 9.548946453464296e-05, "loss": 0.008768947795033455, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00881, "step": 96, "tokens/total": 3081768, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 43623} +{"epoch": 0.37890625, "grad_norm": 0.5564000010490417, "learning_rate": 9.53619478457953e-05, "loss": 0.013005208224058151, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01309, "step": 97, "tokens/total": 3115608, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 44079} +{"epoch": 0.3828125, "grad_norm": 0.1707083284854889, "learning_rate": 9.523275153154695e-05, "loss": 0.003918730653822422, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00393, "step": 98, "tokens/total": 3146880, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 44529} +{"epoch": 0.38671875, "grad_norm": 0.6584495902061462, "learning_rate": 9.51018809682839e-05, "loss": 0.026386888697743416, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.02674, "step": 99, "tokens/total": 3180088, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 44976} +{"epoch": 0.390625, "grad_norm": 0.601332426071167, "learning_rate": 9.49693416020645e-05, "loss": 0.010374191217124462, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.01043, "step": 100, "tokens/total": 3213968, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 45429} +{"epoch": 0.39453125, "grad_norm": 0.37675315141677856, "learning_rate": 9.483513894839276e-05, "loss": 0.003884017700329423, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00389, "step": 101, "tokens/total": 3245976, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 45876} +{"epoch": 0.3984375, "grad_norm": 0.25066232681274414, "learning_rate": 9.469927859198888e-05, "loss": 0.003953683190047741, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00396, "step": 102, "tokens/total": 3278792, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 46363} +{"epoch": 0.40234375, "grad_norm": 4.108895778656006, "learning_rate": 9.456176618655689e-05, "loss": 0.014481928199529648, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.01459, "step": 103, "tokens/total": 3309360, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 46810} +{"epoch": 0.40625, "grad_norm": 1.7841242551803589, "learning_rate": 9.442260745454927e-05, "loss": 0.011553757824003696, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.01162, "step": 104, "tokens/total": 3344344, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 47287} +{"epoch": 0.41015625, "grad_norm": 0.3122326731681824, "learning_rate": 9.428180818692884e-05, "loss": 0.003312664804980159, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00332, "step": 105, "tokens/total": 3378832, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 47788} +{"epoch": 0.4140625, "grad_norm": 0.47667425870895386, "learning_rate": 9.413937424292791e-05, "loss": 0.003058399073779583, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00306, "step": 106, "tokens/total": 3415416, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 48243} +{"epoch": 0.41796875, "grad_norm": 0.7244662046432495, "learning_rate": 9.399531154980424e-05, "loss": 0.02105596289038658, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.02128, "step": 107, "tokens/total": 3448120, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 48692} +{"epoch": 0.421875, "grad_norm": 1.5569336414337158, "learning_rate": 9.384962610259455e-05, "loss": 0.023769522085785866, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.02405, "step": 108, "tokens/total": 3480800, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.6088758707046509, "learning_rate": 9.370232396386494e-05, "loss": 0.004848375916481018, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00486, "step": 109, "tokens/total": 3509992, "tokens/train_per_sec_per_gpu": 10.23, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.931037962436676, "learning_rate": 9.355341126345868e-05, "loss": 0.011666151694953442, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01173, "step": 110, "tokens/total": 3542872, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.17784465849399567, "learning_rate": 9.340289419824107e-05, "loss": 0.0027316734194755554, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.00274, "step": 111, "tokens/total": 3573896, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.7273784875869751, "learning_rate": 9.325077903184159e-05, "loss": 0.006461607292294502, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00648, "step": 112, "tokens/total": 3605648, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.7525582909584045, "learning_rate": 9.30970720943932e-05, "loss": 0.010084227658808231, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.01014, "step": 113, "tokens/total": 3638544, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 51389} +{"epoch": 0.4453125, "grad_norm": 2.8613510131835938, "learning_rate": 9.2941779782269e-05, "loss": 0.01323577668517828, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01332, "step": 114, "tokens/total": 3670776, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 51862} +{"epoch": 0.44921875, "grad_norm": 0.8601484298706055, "learning_rate": 9.278490855781596e-05, "loss": 0.02370566874742508, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.02399, "step": 115, "tokens/total": 3701472, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 52301} +{"epoch": 0.453125, "grad_norm": 0.3341323435306549, "learning_rate": 9.262646494908604e-05, "loss": 0.009792241267859936, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00984, "step": 116, "tokens/total": 3735872, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 52755} +{"epoch": 0.45703125, "grad_norm": 0.9622647762298584, "learning_rate": 9.246645554956457e-05, "loss": 0.025884119793772697, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.02622, "step": 117, "tokens/total": 3770032, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 53224} +{"epoch": 0.4609375, "grad_norm": 0.3923793435096741, "learning_rate": 9.230488701789578e-05, "loss": 0.010701142251491547, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.01076, "step": 118, "tokens/total": 3805264, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 53666} +{"epoch": 0.46484375, "grad_norm": 0.4352741837501526, "learning_rate": 9.214176607760577e-05, "loss": 0.018093567341566086, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.01826, "step": 119, "tokens/total": 3840776, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 54129} +{"epoch": 0.46875, "grad_norm": 0.7869374752044678, "learning_rate": 9.197709951682268e-05, "loss": 0.016580656170845032, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.01672, "step": 120, "tokens/total": 3871376, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 54587} +{"epoch": 0.47265625, "grad_norm": 0.7244143486022949, "learning_rate": 9.181089418799428e-05, "loss": 0.01773746870458126, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.0179, "step": 121, "tokens/total": 3904552, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 55070} +{"epoch": 0.4765625, "grad_norm": 1.6309565305709839, "learning_rate": 9.164315700760271e-05, "loss": 0.006352613214403391, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00637, "step": 122, "tokens/total": 3937344, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 55494} +{"epoch": 0.48046875, "grad_norm": 10.590537071228027, "learning_rate": 9.147389495587671e-05, "loss": 0.02407000958919525, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.02436, "step": 123, "tokens/total": 3967392, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 55940} +{"epoch": 0.484375, "grad_norm": 0.8669401407241821, "learning_rate": 9.130311507650116e-05, "loss": 0.008178836666047573, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00821, "step": 124, "tokens/total": 3998280, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 56404} +{"epoch": 0.48828125, "grad_norm": 0.5710088610649109, "learning_rate": 9.113082447632394e-05, "loss": 0.0117417573928833, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.01181, "step": 125, "tokens/total": 4029616, "tokens/train_per_sec_per_gpu": 10.4, "tokens/trainable": 56878} +{"epoch": 0.4921875, "grad_norm": 0.3580723702907562, "learning_rate": 9.09570303250602e-05, "loss": 0.005238437093794346, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00525, "step": 126, "tokens/total": 4060368, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 57362} +{"epoch": 0.49609375, "grad_norm": 0.41353335976600647, "learning_rate": 9.078173985499394e-05, "loss": 0.0033829379826784134, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00339, "step": 127, "tokens/total": 4089688, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 57792} +{"epoch": 0.5, "grad_norm": 0.8104551434516907, "learning_rate": 9.060496036067713e-05, "loss": 0.011288397945463657, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.01135, "step": 128, "tokens/total": 4125048, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 58216} +{"epoch": 0.50390625, "grad_norm": 0.13456957042217255, "learning_rate": 9.042669919862615e-05, "loss": 0.002424130914732814, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00243, "step": 129, "tokens/total": 4156488, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 58680} +{"epoch": 0.5078125, "grad_norm": 0.5363569259643555, "learning_rate": 9.024696378701557e-05, "loss": 0.007639337796717882, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00767, "step": 130, "tokens/total": 4189344, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 59113} +{"epoch": 0.51171875, "grad_norm": 0.2753864526748657, "learning_rate": 9.006576160536948e-05, "loss": 0.0029105120338499546, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00291, "step": 131, "tokens/total": 4220768, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 59562} +{"epoch": 0.515625, "grad_norm": 0.5082362294197083, "learning_rate": 8.988310019425035e-05, "loss": 0.0008358867489732802, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00084, "step": 132, "tokens/total": 4256880, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 60006} +{"epoch": 0.51953125, "grad_norm": 0.1597948968410492, "learning_rate": 8.969898715494506e-05, "loss": 0.0020024224650114775, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.002, "step": 133, "tokens/total": 4290928, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 60487} +{"epoch": 0.5234375, "grad_norm": 0.4780717194080353, "learning_rate": 8.951343014914869e-05, "loss": 0.015716996043920517, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.01584, "step": 134, "tokens/total": 4325728, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 60953} +{"epoch": 0.52734375, "grad_norm": 0.3918829560279846, "learning_rate": 8.932643689864568e-05, "loss": 0.002298847073689103, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.0023, "step": 135, "tokens/total": 4358848, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 61440} +{"epoch": 0.53125, "grad_norm": 1.1080050468444824, "learning_rate": 8.913801518498845e-05, "loss": 0.012411016970872879, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.01249, "step": 136, "tokens/total": 4393504, "tokens/train_per_sec_per_gpu": 6.84, "tokens/trainable": 61901} +{"epoch": 0.53515625, "grad_norm": 0.9583927392959595, "learning_rate": 8.894817284917364e-05, "loss": 0.005645068362355232, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00566, "step": 137, "tokens/total": 4427912, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 62355} +{"epoch": 0.5390625, "grad_norm": 0.9805415272712708, "learning_rate": 8.875691779131569e-05, "loss": 0.001971925375983119, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00197, "step": 138, "tokens/total": 4462448, "tokens/train_per_sec_per_gpu": 6.34, "tokens/trainable": 62786} +{"epoch": 0.54296875, "grad_norm": 0.8849504590034485, "learning_rate": 8.856425797031829e-05, "loss": 0.018209705129265785, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01838, "step": 139, "tokens/total": 4495992, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 63228} +{"epoch": 0.546875, "grad_norm": 1.5322794914245605, "learning_rate": 8.837020140354295e-05, "loss": 0.013469714671373367, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.01356, "step": 140, "tokens/total": 4529600, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 63693} +{"epoch": 0.55078125, "grad_norm": 5.515050411224365, "learning_rate": 8.817475616647554e-05, "loss": 0.059537820518016815, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.06135, "step": 141, "tokens/total": 4562536, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 64159} +{"epoch": 0.5546875, "grad_norm": 1.8718146085739136, "learning_rate": 8.797793039239017e-05, "loss": 0.01589534990489483, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.01602, "step": 142, "tokens/total": 4595232, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 64608} +{"epoch": 0.55859375, "grad_norm": 3.33536696434021, "learning_rate": 8.777973227201069e-05, "loss": 0.023323923349380493, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.0236, "step": 143, "tokens/total": 4629712, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 65117} +{"epoch": 0.5625, "grad_norm": 0.8161845207214355, "learning_rate": 8.758017005316988e-05, "loss": 0.013544535264372826, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.01364, "step": 144, "tokens/total": 4661752, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 65542} +{"epoch": 0.56640625, "grad_norm": 0.7939583659172058, "learning_rate": 8.737925204046629e-05, "loss": 0.01545409020036459, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.01557, "step": 145, "tokens/total": 4694416, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 65992} +{"epoch": 0.5703125, "grad_norm": 0.5290973782539368, "learning_rate": 8.717698659491851e-05, "loss": 0.012134857475757599, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.01221, "step": 146, "tokens/total": 4728264, "tokens/train_per_sec_per_gpu": 6.97, "tokens/trainable": 66449} +{"epoch": 0.57421875, "grad_norm": 0.2714841365814209, "learning_rate": 8.697338213361735e-05, "loss": 0.007778434082865715, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00781, "step": 147, "tokens/total": 4758952, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 66897} +{"epoch": 0.578125, "grad_norm": 2.9404690265655518, "learning_rate": 8.676844712937552e-05, "loss": 0.005735025741159916, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00575, "step": 148, "tokens/total": 4795624, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 67369} +{"epoch": 0.58203125, "grad_norm": 0.9346358776092529, "learning_rate": 8.656219011037509e-05, "loss": 0.015042352490127087, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01516, "step": 149, "tokens/total": 4828976, "tokens/train_per_sec_per_gpu": 6.57, "tokens/trainable": 67857} +{"epoch": 0.5859375, "grad_norm": 0.29533666372299194, "learning_rate": 8.63546196598125e-05, "loss": 0.005692378152161837, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00571, "step": 150, "tokens/total": 4862128, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 68305} +{"epoch": 0.58984375, "grad_norm": 0.3115580081939697, "learning_rate": 8.614574441554145e-05, "loss": 0.010323434136807919, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.01038, "step": 151, "tokens/total": 4892760, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 68730} +{"epoch": 0.59375, "grad_norm": 0.8190737962722778, "learning_rate": 8.593557306971349e-05, "loss": 0.02107074111700058, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.02129, "step": 152, "tokens/total": 4925296, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 69185} +{"epoch": 0.59765625, "grad_norm": 0.589065670967102, "learning_rate": 8.572411436841618e-05, "loss": 0.006699245888739824, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00672, "step": 153, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 69660} +{"epoch": 0.6015625, "grad_norm": 0.18110235035419464, "learning_rate": 8.551137711130922e-05, "loss": 0.003097234293818474, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.0031, "step": 154, "tokens/total": 4985520, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 70066} +{"epoch": 0.60546875, "grad_norm": 17.708921432495117, "learning_rate": 8.529737015125824e-05, "loss": 0.00646635890007019, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00649, "step": 155, "tokens/total": 5019320, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 70532} +{"epoch": 0.609375, "grad_norm": 0.3679768443107605, "learning_rate": 8.508210239396639e-05, "loss": 0.005936898291110992, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00595, "step": 156, "tokens/total": 5052056, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 70981} +{"epoch": 0.61328125, "grad_norm": 0.2275868058204651, "learning_rate": 8.486558279760375e-05, "loss": 0.00306263891980052, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00307, "step": 157, "tokens/total": 5082280, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 71407} +{"epoch": 0.6171875, "grad_norm": 0.15270604193210602, "learning_rate": 8.464782037243449e-05, "loss": 0.0015510722296312451, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00155, "step": 158, "tokens/total": 5113912, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 71848} +{"epoch": 0.62109375, "grad_norm": 1.0114988088607788, "learning_rate": 8.442882418044202e-05, "loss": 0.02601645514369011, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.02636, "step": 159, "tokens/total": 5145200, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 72325} +{"epoch": 0.625, "grad_norm": 0.46757379174232483, "learning_rate": 8.420860333495179e-05, "loss": 0.007035048212856054, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00706, "step": 160, "tokens/total": 5174928, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 72739} +{"epoch": 0.62890625, "grad_norm": 0.42350900173187256, "learning_rate": 8.398716700025208e-05, "loss": 0.007605843245983124, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00763, "step": 161, "tokens/total": 5206888, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 73191} +{"epoch": 0.6328125, "grad_norm": 3.0767650604248047, "learning_rate": 8.376452439121266e-05, "loss": 0.024062758311629295, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.02435, "step": 162, "tokens/total": 5236336, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 73642} +{"epoch": 0.63671875, "grad_norm": 0.6221123933792114, "learning_rate": 8.354068477290124e-05, "loss": 0.005075424909591675, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00509, "step": 163, "tokens/total": 5267760, "tokens/train_per_sec_per_gpu": 10.08, "tokens/trainable": 74101} +{"epoch": 0.640625, "grad_norm": 0.47906410694122314, "learning_rate": 8.331565746019807e-05, "loss": 0.004409852437674999, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00442, "step": 164, "tokens/total": 5298896, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 74558} +{"epoch": 0.64453125, "grad_norm": 4.134277820587158, "learning_rate": 8.308945181740812e-05, "loss": 0.057101257145404816, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.05876, "step": 165, "tokens/total": 5330792, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 75000} +{"epoch": 0.6484375, "grad_norm": 0.16989478468894958, "learning_rate": 8.286207725787153e-05, "loss": 0.00894707627594471, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00899, "step": 166, "tokens/total": 5364144, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 75445} +{"epoch": 0.65234375, "grad_norm": 1.3886185884475708, "learning_rate": 8.263354324357182e-05, "loss": 0.011864392086863518, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.01194, "step": 167, "tokens/total": 5396656, "tokens/train_per_sec_per_gpu": 6.79, "tokens/trainable": 75880} +{"epoch": 0.65625, "grad_norm": 0.5440099239349365, "learning_rate": 8.240385928474219e-05, "loss": 0.01469430048018694, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.0148, "step": 168, "tokens/total": 5427736, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 76294} +{"epoch": 0.66015625, "grad_norm": 0.43333837389945984, "learning_rate": 8.217303493946967e-05, "loss": 0.007274181582033634, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.0073, "step": 169, "tokens/total": 5461752, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 76742} +{"epoch": 0.6640625, "grad_norm": 0.6001920104026794, "learning_rate": 8.194107981329746e-05, "loss": 0.019328109920024872, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01952, "step": 170, "tokens/total": 5498688, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 77196} +{"epoch": 0.66796875, "grad_norm": 0.08142859488725662, "learning_rate": 8.170800355882518e-05, "loss": 0.0013338588178157806, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00133, "step": 171, "tokens/total": 5530936, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 77663} +{"epoch": 0.671875, "grad_norm": 0.7263751029968262, "learning_rate": 8.147381587530713e-05, "loss": 0.009701108559966087, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00975, "step": 172, "tokens/total": 5564176, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 78117} +{"epoch": 0.67578125, "grad_norm": 0.5038536190986633, "learning_rate": 8.123852650824877e-05, "loss": 0.006845360621809959, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00687, "step": 173, "tokens/total": 5596856, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 78601} +{"epoch": 0.6796875, "grad_norm": 0.5170801877975464, "learning_rate": 8.100214524900103e-05, "loss": 0.011639931239187717, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.01171, "step": 174, "tokens/total": 5630016, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 79032} +{"epoch": 0.68359375, "grad_norm": 0.21341146528720856, "learning_rate": 8.076468193435301e-05, "loss": 0.005227068439126015, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00524, "step": 175, "tokens/total": 5659656, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 79456} +{"epoch": 0.6875, "grad_norm": 0.6813037395477295, "learning_rate": 8.052614644612253e-05, "loss": 0.008006947115063667, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00804, "step": 176, "tokens/total": 5692408, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 79917} +{"epoch": 0.69140625, "grad_norm": 0.469645231962204, "learning_rate": 8.028654871074489e-05, "loss": 0.002382132923230529, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00238, "step": 177, "tokens/total": 5724672, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 80358} +{"epoch": 0.6953125, "grad_norm": 0.2771913707256317, "learning_rate": 8.004589869885986e-05, "loss": 0.0025580557994544506, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00256, "step": 178, "tokens/total": 5754384, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 80777} +{"epoch": 0.69921875, "grad_norm": 1.0361337661743164, "learning_rate": 7.980420642489674e-05, "loss": 0.01982269622385502, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.02002, "step": 179, "tokens/total": 5789344, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 81250} +{"epoch": 0.703125, "grad_norm": 0.19888527691364288, "learning_rate": 7.95614819466576e-05, "loss": 0.0019376499112695456, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00194, "step": 180, "tokens/total": 5821816, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 81715} +{"epoch": 0.70703125, "grad_norm": 0.7687271237373352, "learning_rate": 7.931773536489872e-05, "loss": 0.021294182166457176, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.02152, "step": 181, "tokens/total": 5854168, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 82160} +{"epoch": 0.7109375, "grad_norm": 0.8706613779067993, "learning_rate": 7.907297682291035e-05, "loss": 0.013389816507697105, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01348, "step": 182, "tokens/total": 5886064, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 82624} +{"epoch": 0.71484375, "grad_norm": 0.4283204674720764, "learning_rate": 7.882721650609442e-05, "loss": 0.007397121284157038, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00742, "step": 183, "tokens/total": 5921648, "tokens/train_per_sec_per_gpu": 6.68, "tokens/trainable": 83125} +{"epoch": 0.71875, "grad_norm": 0.7196033000946045, "learning_rate": 7.85804646415409e-05, "loss": 0.013851412571966648, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.01395, "step": 184, "tokens/total": 5954952, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 83581} +{"epoch": 0.72265625, "grad_norm": 0.08517766743898392, "learning_rate": 7.833273149760207e-05, "loss": 0.0014553969958797097, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00146, "step": 185, "tokens/total": 5985432, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 84051} +{"epoch": 0.7265625, "grad_norm": 2.575134754180908, "learning_rate": 7.808402738346527e-05, "loss": 0.02356179989874363, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.02384, "step": 186, "tokens/total": 6020664, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 84500} +{"epoch": 0.73046875, "grad_norm": 0.18365322053432465, "learning_rate": 7.783436264872382e-05, "loss": 0.0012824409641325474, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00128, "step": 187, "tokens/total": 6055024, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 84986} +{"epoch": 0.734375, "grad_norm": 0.15477518737316132, "learning_rate": 7.758374768294647e-05, "loss": 0.0027703566011041403, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00277, "step": 188, "tokens/total": 6088296, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 85465} +{"epoch": 0.73828125, "grad_norm": 0.44151097536087036, "learning_rate": 7.733219291524489e-05, "loss": 0.008918965235352516, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00896, "step": 189, "tokens/total": 6119840, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 85896} +{"epoch": 0.7421875, "grad_norm": 0.28443643450737, "learning_rate": 7.707970881383977e-05, "loss": 0.00366233941167593, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.00367, "step": 190, "tokens/total": 6149880, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 86346} +{"epoch": 0.74609375, "grad_norm": 0.516815721988678, "learning_rate": 7.682630588562518e-05, "loss": 0.006878623738884926, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.0069, "step": 191, "tokens/total": 6182904, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 86777} +{"epoch": 0.75, "grad_norm": 0.3438321650028229, "learning_rate": 7.657199467573129e-05, "loss": 0.006547591183334589, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00657, "step": 192, "tokens/total": 6214144, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 87249} +{"epoch": 0.75390625, "grad_norm": 1.2631373405456543, "learning_rate": 7.631678576708561e-05, "loss": 0.020069357007741928, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.02027, "step": 193, "tokens/total": 6247072, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 87741} +{"epoch": 0.7578125, "grad_norm": 0.22100380063056946, "learning_rate": 7.606068977997255e-05, "loss": 0.001618985435925424, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00162, "step": 194, "tokens/total": 6281112, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 88219} +{"epoch": 0.76171875, "grad_norm": 0.3776657283306122, "learning_rate": 7.580371737159148e-05, "loss": 0.007516722660511732, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00755, "step": 195, "tokens/total": 6310200, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 88630} +{"epoch": 0.765625, "grad_norm": 0.517036497592926, "learning_rate": 7.554587923561324e-05, "loss": 0.011105084791779518, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.01117, "step": 196, "tokens/total": 6341792, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 89068} +{"epoch": 0.76953125, "grad_norm": 1.4735591411590576, "learning_rate": 7.528718610173511e-05, "loss": 0.05205629765987396, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.05344, "step": 197, "tokens/total": 6374040, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 89489} +{"epoch": 0.7734375, "grad_norm": 0.30846744775772095, "learning_rate": 7.502764873523431e-05, "loss": 0.009911478497087955, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00996, "step": 198, "tokens/total": 6405552, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 89928} +{"epoch": 0.77734375, "grad_norm": 0.49804264307022095, "learning_rate": 7.476727793652011e-05, "loss": 0.009921345859766006, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00997, "step": 199, "tokens/total": 6436016, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 90407} +{"epoch": 0.78125, "grad_norm": 0.35275885462760925, "learning_rate": 7.450608454068415e-05, "loss": 0.009086306206882, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00913, "step": 200, "tokens/total": 6468656, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 90849} +{"epoch": 0.78515625, "grad_norm": 0.4016978144645691, "learning_rate": 7.424407941704987e-05, "loss": 0.015150853432714939, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.01527, "step": 201, "tokens/total": 6502728, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 91350} +{"epoch": 0.7890625, "grad_norm": 0.9454311728477478, "learning_rate": 7.398127346871986e-05, "loss": 0.030527157709002495, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.031, "step": 202, "tokens/total": 6535616, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 91805} +{"epoch": 0.79296875, "grad_norm": 0.20619341731071472, "learning_rate": 7.371767763212238e-05, "loss": 0.005448248237371445, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00546, "step": 203, "tokens/total": 6567088, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 92258} +{"epoch": 0.796875, "grad_norm": 0.21402065455913544, "learning_rate": 7.345330287655617e-05, "loss": 0.008142081089317799, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00818, "step": 204, "tokens/total": 6600808, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 92721} +{"epoch": 0.80078125, "grad_norm": 0.24472035467624664, "learning_rate": 7.31881602037339e-05, "loss": 0.005874790251255035, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.00589, "step": 205, "tokens/total": 6631384, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 93160} +{"epoch": 0.8046875, "grad_norm": 0.5180626511573792, "learning_rate": 7.29222606473245e-05, "loss": 0.012873235158622265, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.01296, "step": 206, "tokens/total": 6661888, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 93605} +{"epoch": 0.80859375, "grad_norm": 0.3762667179107666, "learning_rate": 7.265561527249383e-05, "loss": 0.011246979236602783, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.01131, "step": 207, "tokens/total": 6693456, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 94041} +{"epoch": 0.8125, "grad_norm": 17.55360984802246, "learning_rate": 7.238823517544436e-05, "loss": 0.013375631533563137, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01347, "step": 208, "tokens/total": 6727840, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 94512} +{"epoch": 0.81640625, "grad_norm": 0.20482687652111053, "learning_rate": 7.212013148295333e-05, "loss": 0.005003089085221291, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00502, "step": 209, "tokens/total": 6761704, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 94994} +{"epoch": 0.8203125, "grad_norm": 0.2612266540527344, "learning_rate": 7.185131535190975e-05, "loss": 0.004318121820688248, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00433, "step": 210, "tokens/total": 6792648, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 95503} +{"epoch": 0.82421875, "grad_norm": 3.686225414276123, "learning_rate": 7.158179796885005e-05, "loss": 0.0059517752379179, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00597, "step": 211, "tokens/total": 6824424, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 95954} +{"epoch": 0.828125, "grad_norm": 0.104884073138237, "learning_rate": 7.131159054949273e-05, "loss": 0.0022250947076827288, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00223, "step": 212, "tokens/total": 6861216, "tokens/train_per_sec_per_gpu": 7.01, "tokens/trainable": 96428} +{"epoch": 0.83203125, "grad_norm": 0.5195683240890503, "learning_rate": 7.104070433827139e-05, "loss": 0.006558225490152836, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00658, "step": 213, "tokens/total": 6892704, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 96828} +{"epoch": 0.8359375, "grad_norm": 0.42946597933769226, "learning_rate": 7.076915060786705e-05, "loss": 0.017436478286981583, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.01759, "step": 214, "tokens/total": 6924816, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 97270} +{"epoch": 0.83984375, "grad_norm": 0.28600215911865234, "learning_rate": 7.049694065873882e-05, "loss": 0.005123362876474857, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00514, "step": 215, "tokens/total": 6957536, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 97727} +{"epoch": 0.84375, "grad_norm": 0.4552556276321411, "learning_rate": 7.022408581865382e-05, "loss": 0.005185648333281279, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.0052, "step": 216, "tokens/total": 6991448, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 98201} +{"epoch": 0.84765625, "grad_norm": 0.02236669696867466, "learning_rate": 6.99505974422157e-05, "loss": 0.00017771907732822, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00018, "step": 217, "tokens/total": 7023712, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 98674} +{"epoch": 0.8515625, "grad_norm": 0.0680895522236824, "learning_rate": 6.967648691039213e-05, "loss": 0.0007096581975929439, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00071, "step": 218, "tokens/total": 7055040, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 99108} +{"epoch": 0.85546875, "grad_norm": 0.48804065585136414, "learning_rate": 6.940176563004123e-05, "loss": 0.003236344549804926, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00324, "step": 219, "tokens/total": 7084944, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 99540} +{"epoch": 0.859375, "grad_norm": 0.24990636110305786, "learning_rate": 6.912644503343682e-05, "loss": 0.00418633408844471, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.0042, "step": 220, "tokens/total": 7119264, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 99978} +{"epoch": 0.86328125, "grad_norm": 0.9584707617759705, "learning_rate": 6.885053657779273e-05, "loss": 0.012100343592464924, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.01217, "step": 221, "tokens/total": 7150848, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 100450} +{"epoch": 0.8671875, "grad_norm": 1.5744049549102783, "learning_rate": 6.857405174478604e-05, "loss": 0.006569725926965475, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00659, "step": 222, "tokens/total": 7185176, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 100903} +{"epoch": 0.87109375, "grad_norm": 0.8003804683685303, "learning_rate": 6.82970020400792e-05, "loss": 0.022573666647076607, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.02283, "step": 223, "tokens/total": 7219024, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 101351} +{"epoch": 0.875, "grad_norm": 0.07145848125219345, "learning_rate": 6.801939899284132e-05, "loss": 0.00035902991658076644, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00036, "step": 224, "tokens/total": 7253816, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 101857} +{"epoch": 0.87890625, "grad_norm": 0.2867007553577423, "learning_rate": 6.774125415526827e-05, "loss": 0.004152840003371239, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00416, "step": 225, "tokens/total": 7286344, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 102328} +{"epoch": 0.8828125, "grad_norm": 0.13754454255104065, "learning_rate": 6.746257910210214e-05, "loss": 0.0006877190317027271, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00069, "step": 226, "tokens/total": 7317952, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 102779} +{"epoch": 0.88671875, "grad_norm": 0.6988995671272278, "learning_rate": 6.718338543014937e-05, "loss": 0.02391643263399601, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.0242, "step": 227, "tokens/total": 7347336, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 103228} +{"epoch": 0.890625, "grad_norm": 0.03098941594362259, "learning_rate": 6.69036847577983e-05, "loss": 0.0004805884091183543, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00048, "step": 228, "tokens/total": 7377592, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 103684} +{"epoch": 0.89453125, "grad_norm": 0.288143515586853, "learning_rate": 6.662348872453553e-05, "loss": 0.008239364251494408, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00827, "step": 229, "tokens/total": 7410184, "tokens/train_per_sec_per_gpu": 10.35, "tokens/trainable": 104145} +{"epoch": 0.8984375, "grad_norm": 0.11983423680067062, "learning_rate": 6.63428089904618e-05, "loss": 0.0015201896894723177, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00152, "step": 230, "tokens/total": 7443984, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 104598} +{"epoch": 0.90234375, "grad_norm": 0.25795823335647583, "learning_rate": 6.60616572358065e-05, "loss": 0.004372687544673681, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00438, "step": 231, "tokens/total": 7472480, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 104998} +{"epoch": 0.90625, "grad_norm": 0.5566478371620178, "learning_rate": 6.578004516044172e-05, "loss": 0.015356452204287052, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.01547, "step": 232, "tokens/total": 7505992, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 105465} +{"epoch": 0.91015625, "grad_norm": 0.31141623854637146, "learning_rate": 6.549798448339548e-05, "loss": 0.002494330983608961, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.0025, "step": 233, "tokens/total": 7541168, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 105928} +{"epoch": 0.9140625, "grad_norm": 0.030381379649043083, "learning_rate": 6.521548694236384e-05, "loss": 0.0009096460416913033, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00091, "step": 234, "tokens/total": 7574208, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 106365} +{"epoch": 0.91796875, "grad_norm": 0.2663230001926422, "learning_rate": 6.493256429322259e-05, "loss": 0.005920005030930042, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00594, "step": 235, "tokens/total": 7607016, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 106796} +{"epoch": 0.921875, "grad_norm": 0.238115593791008, "learning_rate": 6.464922830953799e-05, "loss": 0.005646577104926109, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00566, "step": 236, "tokens/total": 7641744, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 107227} +{"epoch": 0.92578125, "grad_norm": 0.3300132155418396, "learning_rate": 6.436549078207688e-05, "loss": 0.007156388834118843, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00718, "step": 237, "tokens/total": 7673096, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 107687} +{"epoch": 0.9296875, "grad_norm": 0.12607519328594208, "learning_rate": 6.408136351831592e-05, "loss": 0.002160583855584264, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00216, "step": 238, "tokens/total": 7705344, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 108127} +{"epoch": 0.93359375, "grad_norm": 0.04274985194206238, "learning_rate": 6.379685834195036e-05, "loss": 0.0006148404208943248, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00062, "step": 239, "tokens/total": 7740240, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 108633} +{"epoch": 0.9375, "grad_norm": 0.11961346119642258, "learning_rate": 6.351198709240186e-05, "loss": 0.002950756112113595, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00296, "step": 240, "tokens/total": 7775392, "tokens/train_per_sec_per_gpu": 6.95, "tokens/trainable": 109085} +{"epoch": 0.94140625, "grad_norm": 0.0763462632894516, "learning_rate": 6.32267616243259e-05, "loss": 0.0018207348184660077, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00182, "step": 241, "tokens/total": 7807296, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 109548} +{"epoch": 0.9453125, "grad_norm": 0.25577205419540405, "learning_rate": 6.294119380711849e-05, "loss": 0.0034240365494042635, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00343, "step": 242, "tokens/total": 7838200, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 109975} +{"epoch": 0.94921875, "grad_norm": 0.7778471112251282, "learning_rate": 6.265529552442209e-05, "loss": 0.01863846741616726, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01881, "step": 243, "tokens/total": 7870936, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 110436} +{"epoch": 0.953125, "grad_norm": 0.27898839116096497, "learning_rate": 6.236907867363127e-05, "loss": 0.006952892057597637, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00698, "step": 244, "tokens/total": 7902512, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 110882} +{"epoch": 0.95703125, "grad_norm": 0.08317172527313232, "learning_rate": 6.208255516539749e-05, "loss": 0.0013296998804435134, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00133, "step": 245, "tokens/total": 7933224, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 111355} +{"epoch": 0.9609375, "grad_norm": 0.1234777495265007, "learning_rate": 6.179573692313344e-05, "loss": 0.0006811992498114705, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00068, "step": 246, "tokens/total": 7968456, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 111803} +{"epoch": 0.96484375, "grad_norm": 0.1612028330564499, "learning_rate": 6.150863588251694e-05, "loss": 0.002339770086109638, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00234, "step": 247, "tokens/total": 8003592, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 112270} +{"epoch": 0.96875, "grad_norm": 0.22412975132465363, "learning_rate": 6.122126399099419e-05, "loss": 0.0021443848963826895, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00215, "step": 248, "tokens/total": 8040048, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 112778} +{"epoch": 0.97265625, "grad_norm": 0.3303784728050232, "learning_rate": 6.0933633207282615e-05, "loss": 0.017335323616862297, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.01749, "step": 249, "tokens/total": 8074000, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 113284} +{"epoch": 0.9765625, "grad_norm": 0.0815642774105072, "learning_rate": 6.064575550087316e-05, "loss": 0.0007367653306573629, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00074, "step": 250, "tokens/total": 8108304, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 113722} +{"epoch": 0.98046875, "grad_norm": 0.4674813449382782, "learning_rate": 6.0357642851532245e-05, "loss": 0.0075118690729141235, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00754, "step": 251, "tokens/total": 8143464, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 114213} +{"epoch": 0.984375, "grad_norm": 0.16888095438480377, "learning_rate": 6.0069307248803294e-05, "loss": 0.0017279671737924218, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00173, "step": 252, "tokens/total": 8175576, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 114634} +{"epoch": 0.98828125, "grad_norm": 0.21524855494499207, "learning_rate": 5.9780760691507635e-05, "loss": 0.002881736494600773, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00289, "step": 253, "tokens/total": 8205528, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 115066} +{"epoch": 0.9921875, "grad_norm": 0.4072779417037964, "learning_rate": 5.9492015187245334e-05, "loss": 0.01241425983607769, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.01249, "step": 254, "tokens/total": 8239424, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 115520} +{"epoch": 0.99609375, "grad_norm": 0.06519800424575806, "learning_rate": 5.920308275189541e-05, "loss": 0.0010643834248185158, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00106, "step": 255, "tokens/total": 8270392, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 115921} +{"epoch": 1.0, "grad_norm": 0.37128129601478577, "learning_rate": 5.8913975409115874e-05, "loss": 0.0047212340869009495, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00473, "step": 256, "tokens/total": 8302096, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 116384} +{"epoch": 1.00390625, "grad_norm": 0.1608288288116455, "learning_rate": 5.8624705189843395e-05, "loss": 0.0024084949400275946, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00241, "step": 257, "tokens/total": 8337064, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 116871} +{"epoch": 1.0078125, "grad_norm": 0.18320691585540771, "learning_rate": 5.833528413179249e-05, "loss": 0.0014919920358806849, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00149, "step": 258, "tokens/total": 8372840, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 117369} +{"epoch": 1.01171875, "grad_norm": 0.2964939773082733, "learning_rate": 5.80457242789548e-05, "loss": 0.002695279661566019, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.0027, "step": 259, "tokens/total": 8409728, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 117866} +{"epoch": 1.015625, "grad_norm": 0.18792349100112915, "learning_rate": 5.77560376810977e-05, "loss": 0.0020002948585897684, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.002, "step": 260, "tokens/total": 8442296, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 118330} +{"epoch": 1.01953125, "grad_norm": 0.07649552077054977, "learning_rate": 5.7466236393263005e-05, "loss": 0.0011454869527369738, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00115, "step": 261, "tokens/total": 8474568, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 118776} +{"epoch": 1.0234375, "grad_norm": 0.03979654237627983, "learning_rate": 5.717633247526522e-05, "loss": 0.0005253547569736838, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00053, "step": 262, "tokens/total": 8505520, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 119211} +{"epoch": 1.02734375, "grad_norm": 0.46183139085769653, "learning_rate": 5.688633799118971e-05, "loss": 0.004606863483786583, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00462, "step": 263, "tokens/total": 8535064, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 119616} +{"epoch": 1.03125, "grad_norm": 0.11637325584888458, "learning_rate": 5.659626500889066e-05, "loss": 0.0011215809499844909, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00112, "step": 264, "tokens/total": 8563600, "tokens/train_per_sec_per_gpu": 6.95, "tokens/trainable": 120086} +{"epoch": 1.03515625, "grad_norm": 0.12128936499357224, "learning_rate": 5.6306125599488905e-05, "loss": 0.0012698916252702475, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00127, "step": 265, "tokens/total": 8592648, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 120525} +{"epoch": 1.0390625, "grad_norm": 0.45887383818626404, "learning_rate": 5.601593183686955e-05, "loss": 0.0049589392729103565, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00497, "step": 266, "tokens/total": 8627328, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 121001} +{"epoch": 1.04296875, "grad_norm": 0.6103278994560242, "learning_rate": 5.572569579717961e-05, "loss": 0.002772567793726921, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00278, "step": 267, "tokens/total": 8659760, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 121465} +{"epoch": 1.046875, "grad_norm": 0.06424902379512787, "learning_rate": 5.543542955832538e-05, "loss": 0.0006647819536738098, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00067, "step": 268, "tokens/total": 8696968, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 121908} +{"epoch": 1.05078125, "grad_norm": 0.07762163132429123, "learning_rate": 5.514514519946986e-05, "loss": 0.000551041797734797, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00055, "step": 269, "tokens/total": 8735392, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 122406} +{"epoch": 1.0546875, "grad_norm": 0.01981683261692524, "learning_rate": 5.485485480053015e-05, "loss": 0.0001912787411129102, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00019, "step": 270, "tokens/total": 8769840, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 122812} +{"epoch": 1.05859375, "grad_norm": 0.45203232765197754, "learning_rate": 5.4564570441674645e-05, "loss": 0.003577559720724821, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00358, "step": 271, "tokens/total": 8800648, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 123252} +{"epoch": 1.0625, "grad_norm": 0.06166869029402733, "learning_rate": 5.42743042028204e-05, "loss": 0.0004238700494170189, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00042, "step": 272, "tokens/total": 8832480, "tokens/train_per_sec_per_gpu": 6.77, "tokens/trainable": 123692} +{"epoch": 1.06640625, "grad_norm": 0.32442063093185425, "learning_rate": 5.3984068163130464e-05, "loss": 0.004544784314930439, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00456, "step": 273, "tokens/total": 8864840, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 124162} +{"epoch": 1.0703125, "grad_norm": 0.2725883424282074, "learning_rate": 5.369387440051111e-05, "loss": 0.003081490518525243, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00309, "step": 274, "tokens/total": 8897272, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 124625} +{"epoch": 1.07421875, "grad_norm": 0.0009442548034712672, "learning_rate": 5.340373499110935e-05, "loss": 1.9534289094735868e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00002, "step": 275, "tokens/total": 8930688, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 125119} +{"epoch": 1.078125, "grad_norm": 0.0029735909774899483, "learning_rate": 5.3113662008810304e-05, "loss": 3.8722006138414145e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00004, "step": 276, "tokens/total": 8967216, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 125601} +{"epoch": 1.08203125, "grad_norm": 0.00785745121538639, "learning_rate": 5.282366752473479e-05, "loss": 7.287260086741298e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00007, "step": 277, "tokens/total": 9001368, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 126075} +{"epoch": 1.0859375, "grad_norm": 0.2838473618030548, "learning_rate": 5.2533763606737005e-05, "loss": 0.0021508336067199707, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00215, "step": 278, "tokens/total": 9036808, "tokens/train_per_sec_per_gpu": 6.8, "tokens/trainable": 126491} +{"epoch": 1.08984375, "grad_norm": 0.07576333731412888, "learning_rate": 5.224396231890232e-05, "loss": 0.0004244436277076602, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00042, "step": 279, "tokens/total": 9067840, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 126940} +{"epoch": 1.09375, "grad_norm": 0.7564808130264282, "learning_rate": 5.195427572104522e-05, "loss": 0.004068105015903711, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00408, "step": 280, "tokens/total": 9098936, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 127376} +{"epoch": 1.09765625, "grad_norm": 0.08882421255111694, "learning_rate": 5.166471586820751e-05, "loss": 0.0004885517410002649, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00049, "step": 281, "tokens/total": 9133264, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 127841} +{"epoch": 1.1015625, "grad_norm": 1.3920756578445435, "learning_rate": 5.1375294810156615e-05, "loss": 0.00874748919159174, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00879, "step": 282, "tokens/total": 9165008, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 128288} +{"epoch": 1.10546875, "grad_norm": 0.0018838844262063503, "learning_rate": 5.1086024590884144e-05, "loss": 2.4707635020604357e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00002, "step": 283, "tokens/total": 9197952, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 128737} +{"epoch": 1.109375, "grad_norm": 0.14279748499393463, "learning_rate": 5.079691724810461e-05, "loss": 0.0004362370236776769, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00044, "step": 284, "tokens/total": 9232576, "tokens/train_per_sec_per_gpu": 7.01, "tokens/trainable": 129172} +{"epoch": 1.11328125, "grad_norm": 0.018796371296048164, "learning_rate": 5.0507984812754684e-05, "loss": 0.00011741789057850838, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00012, "step": 285, "tokens/total": 9264568, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 129669} +{"epoch": 1.1171875, "grad_norm": 0.0037362677976489067, "learning_rate": 5.021923930849237e-05, "loss": 4.742047894978896e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00005, "step": 286, "tokens/total": 9296544, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 130155} +{"epoch": 1.12109375, "grad_norm": 0.035129696130752563, "learning_rate": 4.99306927511967e-05, "loss": 4.616233491105959e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00005, "step": 287, "tokens/total": 9327936, "tokens/train_per_sec_per_gpu": 10.28, "tokens/trainable": 130595} +{"epoch": 1.125, "grad_norm": 0.09130442887544632, "learning_rate": 4.964235714846775e-05, "loss": 0.00043925177305936813, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00044, "step": 288, "tokens/total": 9357984, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 131015} +{"epoch": 1.12890625, "grad_norm": 0.0019310744246467948, "learning_rate": 4.9354244499126866e-05, "loss": 1.9929597328882664e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00002, "step": 289, "tokens/total": 9388664, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 131481} +{"epoch": 1.1328125, "grad_norm": 0.5762863159179688, "learning_rate": 4.90663667927174e-05, "loss": 0.004490456078201532, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.0045, "step": 290, "tokens/total": 9420504, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 131930} +{"epoch": 1.13671875, "grad_norm": 0.060446448624134064, "learning_rate": 4.877873600900581e-05, "loss": 0.0003073357802350074, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00031, "step": 291, "tokens/total": 9452664, "tokens/train_per_sec_per_gpu": 6.13, "tokens/trainable": 132388} +{"epoch": 1.140625, "grad_norm": 0.002852385863661766, "learning_rate": 4.849136411748306e-05, "loss": 4.4059936044504866e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00004, "step": 292, "tokens/total": 9488896, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 132860} +{"epoch": 1.14453125, "grad_norm": 0.07148883491754532, "learning_rate": 4.8204263076866574e-05, "loss": 0.0002892129123210907, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00029, "step": 293, "tokens/total": 9522112, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 133360} +{"epoch": 1.1484375, "grad_norm": 0.004414602182805538, "learning_rate": 4.791744483460251e-05, "loss": 2.7100833904114552e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00003, "step": 294, "tokens/total": 9555632, "tokens/train_per_sec_per_gpu": 6.5, "tokens/trainable": 133793} +{"epoch": 1.15234375, "grad_norm": 0.0014412392629310489, "learning_rate": 4.7630921326368736e-05, "loss": 1.1304757208563387e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00001, "step": 295, "tokens/total": 9590904, "tokens/train_per_sec_per_gpu": 6.33, "tokens/trainable": 134218} +{"epoch": 1.15625, "grad_norm": 0.09006451070308685, "learning_rate": 4.7344704475577916e-05, "loss": 0.00044466464896686375, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00044, "step": 296, "tokens/total": 9620288, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 134654} +{"epoch": 1.16015625, "grad_norm": 0.0664348304271698, "learning_rate": 4.705880619288153e-05, "loss": 0.00038166585727594793, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00038, "step": 297, "tokens/total": 9651984, "tokens/train_per_sec_per_gpu": 6.99, "tokens/trainable": 135056} +{"epoch": 1.1640625, "grad_norm": 0.008557253517210484, "learning_rate": 4.677323837567412e-05, "loss": 7.711528451181948e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00008, "step": 298, "tokens/total": 9681368, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 135483} +{"epoch": 1.16796875, "grad_norm": 0.018439706414937973, "learning_rate": 4.6488012907598146e-05, "loss": 0.0001304903271375224, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00013, "step": 299, "tokens/total": 9712872, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 135957} +{"epoch": 1.171875, "grad_norm": 0.14732785522937775, "learning_rate": 4.620314165804964e-05, "loss": 0.0012022851733490825, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.0012, "step": 300, "tokens/total": 9745184, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 136430} +{"epoch": 1.17578125, "grad_norm": 0.08576459437608719, "learning_rate": 4.591863648168407e-05, "loss": 0.0003175839956384152, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00032, "step": 301, "tokens/total": 9778928, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 136896} +{"epoch": 1.1796875, "grad_norm": 0.476235955953598, "learning_rate": 4.5634509217923135e-05, "loss": 0.0033795528579503298, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00339, "step": 302, "tokens/total": 9810752, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 137331} +{"epoch": 1.18359375, "grad_norm": 0.5271583199501038, "learning_rate": 4.535077169046201e-05, "loss": 0.014597390778362751, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.0147, "step": 303, "tokens/total": 9843672, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 137768} +{"epoch": 1.1875, "grad_norm": 1.8177433013916016, "learning_rate": 4.506743570677743e-05, "loss": 0.018896227702498436, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.01908, "step": 304, "tokens/total": 9873752, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 138200} +{"epoch": 1.19140625, "grad_norm": 0.033361177891492844, "learning_rate": 4.478451305763618e-05, "loss": 0.00032616363023407757, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00033, "step": 305, "tokens/total": 9906272, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 138660} +{"epoch": 1.1953125, "grad_norm": 0.006188912317156792, "learning_rate": 4.450201551660454e-05, "loss": 5.4332536819856614e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00005, "step": 306, "tokens/total": 9936296, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 139092} +{"epoch": 1.19921875, "grad_norm": 0.31644299626350403, "learning_rate": 4.4219954839558276e-05, "loss": 0.006181570701301098, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0062, "step": 307, "tokens/total": 9967720, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 139533} +{"epoch": 1.203125, "grad_norm": 1.8047723770141602, "learning_rate": 4.393834276419352e-05, "loss": 0.0057428739964962006, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.00576, "step": 308, "tokens/total": 9997176, "tokens/train_per_sec_per_gpu": 10.64, "tokens/trainable": 140020} +{"epoch": 1.20703125, "grad_norm": 0.03693777322769165, "learning_rate": 4.36571910095382e-05, "loss": 0.00040575931780040264, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00041, "step": 309, "tokens/total": 10031872, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 140465} +{"epoch": 1.2109375, "grad_norm": 0.1512947827577591, "learning_rate": 4.337651127546448e-05, "loss": 0.0007008936954662204, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0007, "step": 310, "tokens/total": 10065240, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 140915} +{"epoch": 1.21484375, "grad_norm": 0.9451867938041687, "learning_rate": 4.3096315242201736e-05, "loss": 0.01602369174361229, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.01615, "step": 311, "tokens/total": 10096640, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 141350} +{"epoch": 1.21875, "grad_norm": 0.03854110836982727, "learning_rate": 4.2816614569850635e-05, "loss": 0.0003144911315757781, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00031, "step": 312, "tokens/total": 10130192, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 141789} +{"epoch": 1.22265625, "grad_norm": 0.2635416090488434, "learning_rate": 4.2537420897897864e-05, "loss": 0.004334499128162861, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00434, "step": 313, "tokens/total": 10164248, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 142266} +{"epoch": 1.2265625, "grad_norm": 0.027021240442991257, "learning_rate": 4.225874584473174e-05, "loss": 0.0005001199897378683, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.0005, "step": 314, "tokens/total": 10194664, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 142704} +{"epoch": 1.23046875, "grad_norm": 0.0582292377948761, "learning_rate": 4.19806010071587e-05, "loss": 0.0006752258632332087, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00068, "step": 315, "tokens/total": 10227768, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 143168} +{"epoch": 1.234375, "grad_norm": 0.4965689778327942, "learning_rate": 4.170299795992081e-05, "loss": 0.0004286673793103546, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00043, "step": 316, "tokens/total": 10260464, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 143615} +{"epoch": 1.23828125, "grad_norm": 0.051566723734140396, "learning_rate": 4.142594825521398e-05, "loss": 0.0006388856563717127, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00064, "step": 317, "tokens/total": 10297528, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 144096} +{"epoch": 1.2421875, "grad_norm": 0.2002488225698471, "learning_rate": 4.114946342220728e-05, "loss": 0.0022149994038045406, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00222, "step": 318, "tokens/total": 10331448, "tokens/train_per_sec_per_gpu": 6.89, "tokens/trainable": 144543} +{"epoch": 1.24609375, "grad_norm": 0.08318468928337097, "learning_rate": 4.087355496656321e-05, "loss": 0.0015884493477642536, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00159, "step": 319, "tokens/total": 10363152, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 145016} +{"epoch": 1.25, "grad_norm": 0.16916386783123016, "learning_rate": 4.05982343699588e-05, "loss": 0.002365819178521633, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00237, "step": 320, "tokens/total": 10397832, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 145511} +{"epoch": 1.25390625, "grad_norm": 0.07668046653270721, "learning_rate": 4.0323513089607876e-05, "loss": 0.0009648409904912114, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00097, "step": 321, "tokens/total": 10431416, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 145978} +{"epoch": 1.2578125, "grad_norm": 0.6499829888343811, "learning_rate": 4.004940255778431e-05, "loss": 0.003670569509267807, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00368, "step": 322, "tokens/total": 10467384, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 146422} +{"epoch": 1.26171875, "grad_norm": 3.2418909072875977, "learning_rate": 3.977591418134619e-05, "loss": 0.005250685848295689, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00526, "step": 323, "tokens/total": 10500792, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 146863} +{"epoch": 1.265625, "grad_norm": 0.2602664530277252, "learning_rate": 3.95030593412612e-05, "loss": 0.004047600086778402, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00406, "step": 324, "tokens/total": 10532592, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 147286} +{"epoch": 1.26953125, "grad_norm": 0.30238279700279236, "learning_rate": 3.923084939213296e-05, "loss": 0.004228596109896898, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00424, "step": 325, "tokens/total": 10562392, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 147708} +{"epoch": 1.2734375, "grad_norm": 0.022322900593280792, "learning_rate": 3.895929566172861e-05, "loss": 0.00041601737029850483, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00042, "step": 326, "tokens/total": 10595416, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 148172} +{"epoch": 1.27734375, "grad_norm": 0.33318644762039185, "learning_rate": 3.868840945050728e-05, "loss": 0.0037600602954626083, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00377, "step": 327, "tokens/total": 10630504, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 148604} +{"epoch": 1.28125, "grad_norm": 0.261341392993927, "learning_rate": 3.841820203114995e-05, "loss": 0.0031457410659641027, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00315, "step": 328, "tokens/total": 10661632, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 149051} +{"epoch": 1.28515625, "grad_norm": 0.09764067828655243, "learning_rate": 3.814868464809027e-05, "loss": 0.0010362103348597884, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00104, "step": 329, "tokens/total": 10691544, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 149453} +{"epoch": 1.2890625, "grad_norm": 0.2670108675956726, "learning_rate": 3.787986851704667e-05, "loss": 0.0020233672112226486, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00203, "step": 330, "tokens/total": 10724768, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 149883} +{"epoch": 1.29296875, "grad_norm": 0.02164819836616516, "learning_rate": 3.7611764824555654e-05, "loss": 0.00020775293523911387, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00021, "step": 331, "tokens/total": 10756008, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 150357} +{"epoch": 1.296875, "grad_norm": 1.5485670566558838, "learning_rate": 3.734438472750619e-05, "loss": 0.002982404548674822, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00299, "step": 332, "tokens/total": 10793152, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 150828} +{"epoch": 1.30078125, "grad_norm": 0.00491632055491209, "learning_rate": 3.707773935267552e-05, "loss": 7.832802657503635e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00008, "step": 333, "tokens/total": 10827624, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 151314} +{"epoch": 1.3046875, "grad_norm": 0.00258105737157166, "learning_rate": 3.68118397962661e-05, "loss": 3.252941678510979e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00003, "step": 334, "tokens/total": 10859712, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 151722} +{"epoch": 1.30859375, "grad_norm": 0.007318438496440649, "learning_rate": 3.654669712344384e-05, "loss": 5.830618101754226e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00006, "step": 335, "tokens/total": 10888792, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 152178} +{"epoch": 1.3125, "grad_norm": 0.09136597067117691, "learning_rate": 3.628232236787763e-05, "loss": 0.0003424829337745905, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00034, "step": 336, "tokens/total": 10922968, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 152617} +{"epoch": 1.31640625, "grad_norm": 0.006045772228389978, "learning_rate": 3.6018726531280144e-05, "loss": 7.198247476480901e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00007, "step": 337, "tokens/total": 10958000, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 153108} +{"epoch": 1.3203125, "grad_norm": 0.010794644244015217, "learning_rate": 3.575592058295017e-05, "loss": 5.318237890605815e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00005, "step": 338, "tokens/total": 10989872, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 153589} +{"epoch": 1.32421875, "grad_norm": 0.05583810806274414, "learning_rate": 3.549391545931585e-05, "loss": 0.00013038533506914973, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00013, "step": 339, "tokens/total": 11020784, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 154049} +{"epoch": 1.328125, "grad_norm": 0.0015104432823136449, "learning_rate": 3.5232722063479914e-05, "loss": 2.1187623133300804e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00002, "step": 340, "tokens/total": 11053240, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 154498} +{"epoch": 1.33203125, "grad_norm": 0.07925937324762344, "learning_rate": 3.49723512647657e-05, "loss": 0.0006113838753663003, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00061, "step": 341, "tokens/total": 11089040, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 154998} +{"epoch": 1.3359375, "grad_norm": 0.03472711145877838, "learning_rate": 3.471281389826491e-05, "loss": 0.000150095991557464, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00015, "step": 342, "tokens/total": 11119672, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 155403} +{"epoch": 1.33984375, "grad_norm": 0.005381815601140261, "learning_rate": 3.4454120764386764e-05, "loss": 3.161186759825796e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00003, "step": 343, "tokens/total": 11155040, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 155858} +{"epoch": 1.34375, "grad_norm": 0.4064013957977295, "learning_rate": 3.4196282628408526e-05, "loss": 0.015236522071063519, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01535, "step": 344, "tokens/total": 11185952, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 156303} +{"epoch": 1.34765625, "grad_norm": 0.002295290119946003, "learning_rate": 3.3939310220027456e-05, "loss": 3.768468741327524e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00004, "step": 345, "tokens/total": 11219272, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 156798} +{"epoch": 1.3515625, "grad_norm": 0.002273246878758073, "learning_rate": 3.3683214232914404e-05, "loss": 2.5228851882275194e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00003, "step": 346, "tokens/total": 11253424, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 157271} +{"epoch": 1.35546875, "grad_norm": 0.010511163622140884, "learning_rate": 3.342800532426873e-05, "loss": 7.53546628402546e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00008, "step": 347, "tokens/total": 11286216, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 157737} +{"epoch": 1.359375, "grad_norm": 1.1761467456817627, "learning_rate": 3.317369411437484e-05, "loss": 0.005291116889566183, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00531, "step": 348, "tokens/total": 11319480, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 158205} +{"epoch": 1.36328125, "grad_norm": 0.013910269364714622, "learning_rate": 3.292029118616024e-05, "loss": 0.00015400591655634344, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00015, "step": 349, "tokens/total": 11350472, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 158628} +{"epoch": 1.3671875, "grad_norm": 0.016131144016981125, "learning_rate": 3.266780708475511e-05, "loss": 0.00016052593127824366, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00016, "step": 350, "tokens/total": 11383088, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 159106} +{"epoch": 1.37109375, "grad_norm": 0.003795677563175559, "learning_rate": 3.241625231705354e-05, "loss": 5.6720593420322984e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00006, "step": 351, "tokens/total": 11418392, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 159565} +{"epoch": 1.375, "grad_norm": 0.08116691559553146, "learning_rate": 3.216563735127618e-05, "loss": 0.00032581103732809424, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00033, "step": 352, "tokens/total": 11449784, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 160042} +{"epoch": 1.37890625, "grad_norm": 0.6817997097969055, "learning_rate": 3.191597261653475e-05, "loss": 0.0012412877986207604, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00124, "step": 353, "tokens/total": 11484240, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 160502} +{"epoch": 1.3828125, "grad_norm": 0.33786502480506897, "learning_rate": 3.166726850239794e-05, "loss": 0.004863979760557413, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00488, "step": 354, "tokens/total": 11513360, "tokens/train_per_sec_per_gpu": 10.83, "tokens/trainable": 160950} +{"epoch": 1.38671875, "grad_norm": 0.03887750208377838, "learning_rate": 3.141953535845912e-05, "loss": 0.00020536158990580589, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00021, "step": 355, "tokens/total": 11543112, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 161400} +{"epoch": 1.390625, "grad_norm": 0.016939671710133553, "learning_rate": 3.11727834939056e-05, "loss": 0.0001867863757070154, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00019, "step": 356, "tokens/total": 11573464, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 161863} +{"epoch": 1.39453125, "grad_norm": 0.13366365432739258, "learning_rate": 3.092702317708967e-05, "loss": 0.0011094467481598258, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00111, "step": 357, "tokens/total": 11604896, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 162319} +{"epoch": 1.3984375, "grad_norm": 0.015088762156665325, "learning_rate": 3.0682264635101276e-05, "loss": 0.00016206511645577848, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00016, "step": 358, "tokens/total": 11636368, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 162771} +{"epoch": 1.40234375, "grad_norm": 1.644878625869751, "learning_rate": 3.0438518053342407e-05, "loss": 0.00316243851557374, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00317, "step": 359, "tokens/total": 11670216, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 163243} +{"epoch": 1.40625, "grad_norm": 0.28003209829330444, "learning_rate": 3.0195793575103266e-05, "loss": 0.003191571682691574, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.0032, "step": 360, "tokens/total": 11707816, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 163707} +{"epoch": 1.41015625, "grad_norm": 0.19192029535770416, "learning_rate": 2.9954101301140146e-05, "loss": 0.0015984741039574146, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.0016, "step": 361, "tokens/total": 11742584, "tokens/train_per_sec_per_gpu": 6.81, "tokens/trainable": 164157} +{"epoch": 1.4140625, "grad_norm": 0.00514350738376379, "learning_rate": 2.9713451289255123e-05, "loss": 9.352383494842798e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00009, "step": 362, "tokens/total": 11770384, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 164598} +{"epoch": 1.41796875, "grad_norm": 0.006405137944966555, "learning_rate": 2.9473853553877484e-05, "loss": 3.6430428735911846e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00004, "step": 363, "tokens/total": 11803208, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 165072} +{"epoch": 1.421875, "grad_norm": 0.017120346426963806, "learning_rate": 2.9235318065647e-05, "loss": 0.00016505413805134594, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00017, "step": 364, "tokens/total": 11840480, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 165533} +{"epoch": 1.42578125, "grad_norm": 0.01848379522562027, "learning_rate": 2.8997854750998964e-05, "loss": 0.00015795773651916534, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00016, "step": 365, "tokens/total": 11875544, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 165998} +{"epoch": 1.4296875, "grad_norm": 0.06956809014081955, "learning_rate": 2.8761473491751258e-05, "loss": 0.0002777174231596291, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00028, "step": 366, "tokens/total": 11907656, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 166433} +{"epoch": 1.43359375, "grad_norm": 0.1603279411792755, "learning_rate": 2.8526184124692883e-05, "loss": 0.0015095394337549806, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00151, "step": 367, "tokens/total": 11941904, "tokens/train_per_sec_per_gpu": 6.32, "tokens/trainable": 166871} +{"epoch": 1.4375, "grad_norm": 0.4096875786781311, "learning_rate": 2.829199644117484e-05, "loss": 0.0026827992405742407, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00269, "step": 368, "tokens/total": 11973624, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 167281} +{"epoch": 1.44140625, "grad_norm": 0.038898054510354996, "learning_rate": 2.8058920186702553e-05, "loss": 0.00020611501531675458, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00021, "step": 369, "tokens/total": 12010360, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 167754} +{"epoch": 1.4453125, "grad_norm": 0.0021546839270740747, "learning_rate": 2.782696506053033e-05, "loss": 3.564088183338754e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00004, "step": 370, "tokens/total": 12044544, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 168186} +{"epoch": 1.44921875, "grad_norm": 0.13010446727275848, "learning_rate": 2.7596140715257824e-05, "loss": 0.001264776336029172, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00127, "step": 371, "tokens/total": 12075792, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 168651} +{"epoch": 1.453125, "grad_norm": 0.006227811332792044, "learning_rate": 2.7366456756428184e-05, "loss": 5.5613847507629544e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00006, "step": 372, "tokens/total": 12108088, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 169108} +{"epoch": 1.45703125, "grad_norm": 0.005763872992247343, "learning_rate": 2.7137922742128486e-05, "loss": 7.514808385167271e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00008, "step": 373, "tokens/total": 12140888, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 169607} +{"epoch": 1.4609375, "grad_norm": 0.11473281681537628, "learning_rate": 2.691054818259188e-05, "loss": 0.0006145157967694104, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00061, "step": 374, "tokens/total": 12174560, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 170080} +{"epoch": 1.46484375, "grad_norm": 0.1913197636604309, "learning_rate": 2.6684342539801933e-05, "loss": 0.0008010012097656727, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.0008, "step": 375, "tokens/total": 12204848, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 170532} +{"epoch": 1.46875, "grad_norm": 0.01218852587044239, "learning_rate": 2.645931522709877e-05, "loss": 8.705061190994456e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00009, "step": 376, "tokens/total": 12238472, "tokens/train_per_sec_per_gpu": 10.61, "tokens/trainable": 171012} +{"epoch": 1.47265625, "grad_norm": 0.4702986478805542, "learning_rate": 2.6235475608787365e-05, "loss": 0.012477787211537361, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.01256, "step": 377, "tokens/total": 12271024, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 171473} +{"epoch": 1.4765625, "grad_norm": 0.004648304544389248, "learning_rate": 2.6012832999747916e-05, "loss": 8.038378291530535e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00008, "step": 378, "tokens/total": 12301312, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 171920} +{"epoch": 1.48046875, "grad_norm": 0.3617454171180725, "learning_rate": 2.579139666504821e-05, "loss": 0.018657859414815903, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01883, "step": 379, "tokens/total": 12337072, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 172366} +{"epoch": 1.484375, "grad_norm": 0.659967839717865, "learning_rate": 2.557117581955798e-05, "loss": 0.0033181419130414724, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00332, "step": 380, "tokens/total": 12368000, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 172832} +{"epoch": 1.48828125, "grad_norm": 0.012191466055810452, "learning_rate": 2.5352179627565532e-05, "loss": 0.0001907657424453646, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00019, "step": 381, "tokens/total": 12399880, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 173272} +{"epoch": 1.4921875, "grad_norm": 0.3626956045627594, "learning_rate": 2.5134417202396277e-05, "loss": 0.0032785870134830475, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00328, "step": 382, "tokens/total": 12432288, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 173679} +{"epoch": 1.49609375, "grad_norm": 0.026583973318338394, "learning_rate": 2.491789760603361e-05, "loss": 0.0002602323947940022, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00026, "step": 383, "tokens/total": 12465592, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 174109} +{"epoch": 1.5, "grad_norm": 0.01921766810119152, "learning_rate": 2.4702629848741764e-05, "loss": 0.00021532049868255854, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00022, "step": 384, "tokens/total": 12498200, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 174564} +{"epoch": 1.50390625, "grad_norm": 0.4802733063697815, "learning_rate": 2.4488622888690785e-05, "loss": 0.0016522135119885206, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00165, "step": 385, "tokens/total": 12530280, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 175004} +{"epoch": 1.5078125, "grad_norm": 0.013012919574975967, "learning_rate": 2.427588563158384e-05, "loss": 0.00011298443132545799, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00011, "step": 386, "tokens/total": 12563360, "tokens/train_per_sec_per_gpu": 7.08, "tokens/trainable": 175490} +{"epoch": 1.51171875, "grad_norm": 0.07629730552434921, "learning_rate": 2.406442693028651e-05, "loss": 0.0005355161847546697, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00054, "step": 387, "tokens/total": 12593032, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 175911} +{"epoch": 1.515625, "grad_norm": 0.04982919618487358, "learning_rate": 2.3854255584458547e-05, "loss": 0.0003395325911697, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00034, "step": 388, "tokens/total": 12628272, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 176338} +{"epoch": 1.51953125, "grad_norm": 0.9691978096961975, "learning_rate": 2.3645380340187508e-05, "loss": 0.01230169739574194, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.01238, "step": 389, "tokens/total": 12661368, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 176819} +{"epoch": 1.5234375, "grad_norm": 0.07957843691110611, "learning_rate": 2.3437809889624914e-05, "loss": 0.00042501967982389033, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.77, "memory/max_allocated (GiB)": 63.77, "ppl": 1.00043, "step": 390, "tokens/total": 12691168, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 177270} +{"epoch": 1.52734375, "grad_norm": 0.008198228664696217, "learning_rate": 2.3231552870624487e-05, "loss": 0.00012562125630211085, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00013, "step": 391, "tokens/total": 12721264, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 177696} +{"epoch": 1.53125, "grad_norm": 0.24961721897125244, "learning_rate": 2.3026617866382657e-05, "loss": 0.0012972606346011162, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.0013, "step": 392, "tokens/total": 12753640, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 178193} +{"epoch": 1.53515625, "grad_norm": 0.008336912840604782, "learning_rate": 2.2823013405081507e-05, "loss": 0.00015887635527178645, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00016, "step": 393, "tokens/total": 12787056, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 178690} +{"epoch": 1.5390625, "grad_norm": 0.021868785843253136, "learning_rate": 2.2620747959533722e-05, "loss": 0.00016267473984044045, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00016, "step": 394, "tokens/total": 12819312, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 179187} +{"epoch": 1.54296875, "grad_norm": 0.010818032547831535, "learning_rate": 2.2419829946830123e-05, "loss": 0.00011113712389487773, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00011, "step": 395, "tokens/total": 12853376, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 179638} +{"epoch": 1.546875, "grad_norm": 0.005619977600872517, "learning_rate": 2.2220267727989325e-05, "loss": 0.0001326768979197368, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00013, "step": 396, "tokens/total": 12882960, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 180091} +{"epoch": 1.55078125, "grad_norm": 0.08893348276615143, "learning_rate": 2.202206960760984e-05, "loss": 0.000797128421254456, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.0008, "step": 397, "tokens/total": 12917016, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 180598} +{"epoch": 1.5546875, "grad_norm": 0.8019569516181946, "learning_rate": 2.182524383352446e-05, "loss": 0.009221522137522697, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00926, "step": 398, "tokens/total": 12950056, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 181051} +{"epoch": 1.55859375, "grad_norm": 0.03528670594096184, "learning_rate": 2.1629798596457056e-05, "loss": 0.00032228732015937567, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00032, "step": 399, "tokens/total": 12981728, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 181501} +{"epoch": 1.5625, "grad_norm": 0.5727077126502991, "learning_rate": 2.1435742029681725e-05, "loss": 0.005166642367839813, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00518, "step": 400, "tokens/total": 13011464, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 181918} +{"epoch": 1.56640625, "grad_norm": 0.0044508762657642365, "learning_rate": 2.124308220868431e-05, "loss": 7.483711669920012e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00007, "step": 401, "tokens/total": 13044432, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 182373} +{"epoch": 1.5703125, "grad_norm": 0.03612783923745155, "learning_rate": 2.105182715082638e-05, "loss": 0.0005337373586371541, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00053, "step": 402, "tokens/total": 13075608, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 182850} +{"epoch": 1.57421875, "grad_norm": 0.2737535834312439, "learning_rate": 2.0861984815011552e-05, "loss": 0.001695748302154243, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.0017, "step": 403, "tokens/total": 13107352, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 183281} +{"epoch": 1.578125, "grad_norm": 0.006726252846419811, "learning_rate": 2.0673563101354323e-05, "loss": 0.00013400052557699382, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00013, "step": 404, "tokens/total": 13140736, "tokens/train_per_sec_per_gpu": 7.12, "tokens/trainable": 183711} +{"epoch": 1.58203125, "grad_norm": 0.0883559137582779, "learning_rate": 2.0486569850851317e-05, "loss": 0.0006176005699671805, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00062, "step": 405, "tokens/total": 13174040, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 184174} +{"epoch": 1.5859375, "grad_norm": 0.0770566314458847, "learning_rate": 2.0301012845054956e-05, "loss": 0.0009807967580854893, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00098, "step": 406, "tokens/total": 13207232, "tokens/train_per_sec_per_gpu": 6.58, "tokens/trainable": 184589} +{"epoch": 1.58984375, "grad_norm": 0.018678106367588043, "learning_rate": 2.011689980574966e-05, "loss": 0.00028982068761251867, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00029, "step": 407, "tokens/total": 13241160, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 185009} +{"epoch": 1.59375, "grad_norm": 0.02191249094903469, "learning_rate": 1.993423839463052e-05, "loss": 0.0003432975790929049, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00034, "step": 408, "tokens/total": 13273104, "tokens/train_per_sec_per_gpu": 10.32, "tokens/trainable": 185466} +{"epoch": 1.59765625, "grad_norm": 0.17914032936096191, "learning_rate": 1.975303621298445e-05, "loss": 0.0015492856036871672, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00155, "step": 409, "tokens/total": 13308136, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 185918} +{"epoch": 1.6015625, "grad_norm": 0.1370537281036377, "learning_rate": 1.957330080137385e-05, "loss": 0.0014627235941588879, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00146, "step": 410, "tokens/total": 13337816, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 186369} +{"epoch": 1.60546875, "grad_norm": 0.06073486804962158, "learning_rate": 1.9395039639322864e-05, "loss": 0.0002796610933728516, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00028, "step": 411, "tokens/total": 13372576, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 186844} +{"epoch": 1.609375, "grad_norm": 0.003611989552155137, "learning_rate": 1.9218260145006073e-05, "loss": 6.92683897796087e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00007, "step": 412, "tokens/total": 13406664, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 187330} +{"epoch": 1.61328125, "grad_norm": 0.028309186920523643, "learning_rate": 1.904296967493982e-05, "loss": 0.00013337141717784107, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00013, "step": 413, "tokens/total": 13438256, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 187786} +{"epoch": 1.6171875, "grad_norm": 0.13361622393131256, "learning_rate": 1.8869175523676064e-05, "loss": 0.0017192226368933916, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00172, "step": 414, "tokens/total": 13472096, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 188281} +{"epoch": 1.62109375, "grad_norm": 0.08408640325069427, "learning_rate": 1.869688492349885e-05, "loss": 0.0007332442910410464, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00073, "step": 415, "tokens/total": 13504192, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 188733} +{"epoch": 1.625, "grad_norm": 1.1399776935577393, "learning_rate": 1.85261050441233e-05, "loss": 0.00046406424371525645, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00046, "step": 416, "tokens/total": 13537392, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 189189} +{"epoch": 1.62890625, "grad_norm": 0.05760718137025833, "learning_rate": 1.8356842992397304e-05, "loss": 0.0006975694559514523, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.0007, "step": 417, "tokens/total": 13569344, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 189629} +{"epoch": 1.6328125, "grad_norm": 0.3132255971431732, "learning_rate": 1.8189105812005714e-05, "loss": 0.004721313714981079, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00473, "step": 418, "tokens/total": 13602200, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 190039} +{"epoch": 1.63671875, "grad_norm": 0.07935380190610886, "learning_rate": 1.802290048317732e-05, "loss": 0.0005507701425813138, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00055, "step": 419, "tokens/total": 13632640, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 190457} +{"epoch": 1.640625, "grad_norm": 0.01313475426286459, "learning_rate": 1.785823392239424e-05, "loss": 0.00010645866132108495, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00011, "step": 420, "tokens/total": 13664920, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 190898} +{"epoch": 1.64453125, "grad_norm": 0.08158629387617111, "learning_rate": 1.7695112982104225e-05, "loss": 0.0005799854989163578, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00058, "step": 421, "tokens/total": 13697144, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 191337} +{"epoch": 1.6484375, "grad_norm": 0.2050303965806961, "learning_rate": 1.7533544450435433e-05, "loss": 0.0029476406052708626, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00295, "step": 422, "tokens/total": 13724496, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 191770} +{"epoch": 1.65234375, "grad_norm": 0.004162840079516172, "learning_rate": 1.7373535050913946e-05, "loss": 4.659542173612863e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00005, "step": 423, "tokens/total": 13757880, "tokens/train_per_sec_per_gpu": 7.08, "tokens/trainable": 192207} +{"epoch": 1.65625, "grad_norm": 0.33138370513916016, "learning_rate": 1.721509144218405e-05, "loss": 0.005980428773909807, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.006, "step": 424, "tokens/total": 13787648, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 192637} +{"epoch": 1.66015625, "grad_norm": 0.0045076822862029076, "learning_rate": 1.705822021773101e-05, "loss": 8.155960676958784e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00008, "step": 425, "tokens/total": 13819720, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 193123} +{"epoch": 1.6640625, "grad_norm": 0.2573278248310089, "learning_rate": 1.69029279056068e-05, "loss": 0.0007889047847129405, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00079, "step": 426, "tokens/total": 13854944, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 193564} +{"epoch": 1.66796875, "grad_norm": 0.15717975795269012, "learning_rate": 1.6749220968158415e-05, "loss": 0.00035735510755330324, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00036, "step": 427, "tokens/total": 13887824, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 194070} +{"epoch": 1.671875, "grad_norm": 0.12993960082530975, "learning_rate": 1.659710580175893e-05, "loss": 0.0007799735758453608, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00078, "step": 428, "tokens/total": 13918192, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 194569} +{"epoch": 1.67578125, "grad_norm": 0.0258179884403944, "learning_rate": 1.644658873654133e-05, "loss": 0.0002830546582117677, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00028, "step": 429, "tokens/total": 13947440, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 195006} +{"epoch": 1.6796875, "grad_norm": 0.006208243779838085, "learning_rate": 1.629767603613508e-05, "loss": 8.122600411297753e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00008, "step": 430, "tokens/total": 13974360, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 195406} +{"epoch": 1.68359375, "grad_norm": 0.11119364947080612, "learning_rate": 1.615037389740547e-05, "loss": 0.0012208105763420463, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00122, "step": 431, "tokens/total": 14006592, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 195850} +{"epoch": 1.6875, "grad_norm": 0.11597444862127304, "learning_rate": 1.600468845019576e-05, "loss": 0.0007715234532952309, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00077, "step": 432, "tokens/total": 14040776, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 196330} +{"epoch": 1.69140625, "grad_norm": 0.0006413329392671585, "learning_rate": 1.5860625757072092e-05, "loss": 1.1251864634687081e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00001, "step": 433, "tokens/total": 14073040, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 196778} +{"epoch": 1.6953125, "grad_norm": 0.005568118765950203, "learning_rate": 1.571819181307116e-05, "loss": 0.00010414540156489238, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.0001, "step": 434, "tokens/total": 14107624, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 197250} +{"epoch": 1.69921875, "grad_norm": 8.803357124328613, "learning_rate": 1.557739254545075e-05, "loss": 0.005213970318436623, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00523, "step": 435, "tokens/total": 14137784, "tokens/train_per_sec_per_gpu": 10.47, "tokens/trainable": 197722} +{"epoch": 1.703125, "grad_norm": 0.024588748812675476, "learning_rate": 1.543823381344311e-05, "loss": 0.00015250199066940695, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00015, "step": 436, "tokens/total": 14167776, "tokens/train_per_sec_per_gpu": 10.6, "tokens/trainable": 198196} +{"epoch": 1.70703125, "grad_norm": 0.012193465605378151, "learning_rate": 1.5300721408011114e-05, "loss": 0.00021332901087589562, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00021, "step": 437, "tokens/total": 14199752, "tokens/train_per_sec_per_gpu": 10.1, "tokens/trainable": 198653} +{"epoch": 1.7109375, "grad_norm": 0.019361181184649467, "learning_rate": 1.5164861051607254e-05, "loss": 0.00027901495923288167, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.00028, "step": 438, "tokens/total": 14230344, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 199061} +{"epoch": 1.71484375, "grad_norm": 0.011250528506934643, "learning_rate": 1.5030658397935521e-05, "loss": 0.00014013226609677076, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00014, "step": 439, "tokens/total": 14263808, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 199530} +{"epoch": 1.71875, "grad_norm": 0.036273736506700516, "learning_rate": 1.4898119031716104e-05, "loss": 0.0001506915141362697, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00015, "step": 440, "tokens/total": 14295864, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 199988} +{"epoch": 1.72265625, "grad_norm": 0.0334390364587307, "learning_rate": 1.476724846845306e-05, "loss": 0.00031122443033382297, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00031, "step": 441, "tokens/total": 14331688, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 200465} +{"epoch": 1.7265625, "grad_norm": 0.011000513099133968, "learning_rate": 1.463805215420471e-05, "loss": 0.00010298586857970804, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.0001, "step": 442, "tokens/total": 14364688, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 200935} +{"epoch": 1.73046875, "grad_norm": 0.0058736675418913364, "learning_rate": 1.451053546535705e-05, "loss": 6.115723954280838e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00006, "step": 443, "tokens/total": 14397832, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 201391} +{"epoch": 1.734375, "grad_norm": 0.013906121253967285, "learning_rate": 1.438470370840001e-05, "loss": 0.0001122789180953987, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00011, "step": 444, "tokens/total": 14431864, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 201863} +{"epoch": 1.73828125, "grad_norm": 0.005293893162161112, "learning_rate": 1.4260562119706606e-05, "loss": 4.74557928100694e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00005, "step": 445, "tokens/total": 14462152, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 202337} +{"epoch": 1.7421875, "grad_norm": 0.07830771058797836, "learning_rate": 1.413811586531508e-05, "loss": 0.0007982467068359256, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.0008, "step": 446, "tokens/total": 14493072, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 202778} +{"epoch": 1.74609375, "grad_norm": 0.1474275439977646, "learning_rate": 1.4017370040713884e-05, "loss": 0.00035681057488545775, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00036, "step": 447, "tokens/total": 14527392, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 203227} +{"epoch": 1.75, "grad_norm": 0.044044334441423416, "learning_rate": 1.3898329670629645e-05, "loss": 0.0002818532520905137, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00028, "step": 448, "tokens/total": 14561992, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 203672} +{"epoch": 1.75390625, "grad_norm": 0.002253689570352435, "learning_rate": 1.3780999708818058e-05, "loss": 2.379278521402739e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00002, "step": 449, "tokens/total": 14592056, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 204122} +{"epoch": 1.7578125, "grad_norm": 0.16629409790039062, "learning_rate": 1.3665385037857758e-05, "loss": 0.0008752279682084918, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00088, "step": 450, "tokens/total": 14622816, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 204575} +{"epoch": 1.76171875, "grad_norm": 0.006653959397226572, "learning_rate": 1.3551490468947126e-05, "loss": 6.413093069568276e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00006, "step": 451, "tokens/total": 14653272, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 205021} +{"epoch": 1.765625, "grad_norm": 0.9841685891151428, "learning_rate": 1.3439320741704075e-05, "loss": 0.014462817460298538, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.01457, "step": 452, "tokens/total": 14683952, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 205444} +{"epoch": 1.76953125, "grad_norm": 0.1546017974615097, "learning_rate": 1.3328880523968808e-05, "loss": 0.002020779764279723, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00202, "step": 453, "tokens/total": 14716256, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 205907} +{"epoch": 1.7734375, "grad_norm": 0.02146237902343273, "learning_rate": 1.3220174411609587e-05, "loss": 0.00012921115558128804, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00013, "step": 454, "tokens/total": 14749088, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 206346} +{"epoch": 1.77734375, "grad_norm": 0.013245930895209312, "learning_rate": 1.3113206928331471e-05, "loss": 0.0001129750453401357, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00011, "step": 455, "tokens/total": 14777384, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 206808} +{"epoch": 1.78125, "grad_norm": 0.003534448565915227, "learning_rate": 1.300798252548806e-05, "loss": 5.735015292884782e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00006, "step": 456, "tokens/total": 14809640, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 207260} +{"epoch": 1.78515625, "grad_norm": 0.0030020924750715494, "learning_rate": 1.2904505581896265e-05, "loss": 2.6391073333797976e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00003, "step": 457, "tokens/total": 14843728, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 207733} +{"epoch": 1.7890625, "grad_norm": 0.023304328322410583, "learning_rate": 1.2802780403654082e-05, "loss": 0.00029477025964297354, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00029, "step": 458, "tokens/total": 14876256, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 208180} +{"epoch": 1.79296875, "grad_norm": 0.00938305351883173, "learning_rate": 1.2702811223961408e-05, "loss": 0.00012970538227818906, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00013, "step": 459, "tokens/total": 14907000, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 208641} +{"epoch": 1.796875, "grad_norm": 0.09616425633430481, "learning_rate": 1.2604602202943861e-05, "loss": 0.0009560371399857104, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00096, "step": 460, "tokens/total": 14937704, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 209084} +{"epoch": 1.80078125, "grad_norm": 0.0188381876796484, "learning_rate": 1.2508157427479686e-05, "loss": 0.0002020203392021358, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.0002, "step": 461, "tokens/total": 14966920, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 209530} +{"epoch": 1.8046875, "grad_norm": 0.005901062861084938, "learning_rate": 1.2413480911029655e-05, "loss": 8.052119665080681e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00008, "step": 462, "tokens/total": 14996040, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 209969} +{"epoch": 1.80859375, "grad_norm": 0.0020677850116044283, "learning_rate": 1.2320576593470082e-05, "loss": 2.8510963602457196e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00003, "step": 463, "tokens/total": 15029096, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 210396} +{"epoch": 1.8125, "grad_norm": 0.125377357006073, "learning_rate": 1.2229448340928828e-05, "loss": 0.0007188469171524048, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00072, "step": 464, "tokens/total": 15063696, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 210904} +{"epoch": 1.81640625, "grad_norm": 0.0019453096901997924, "learning_rate": 1.2140099945624458e-05, "loss": 3.387439210200682e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00003, "step": 465, "tokens/total": 15098264, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 211362} +{"epoch": 1.8203125, "grad_norm": 0.019929111003875732, "learning_rate": 1.205253512570841e-05, "loss": 0.00010141752136405557, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.0001, "step": 466, "tokens/total": 15130464, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 211810} +{"epoch": 1.82421875, "grad_norm": 0.010959488339722157, "learning_rate": 1.1966757525110255e-05, "loss": 9.391862113261595e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00009, "step": 467, "tokens/total": 15158736, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 212261} +{"epoch": 1.828125, "grad_norm": 0.16086173057556152, "learning_rate": 1.1882770713386095e-05, "loss": 0.0012426411267369986, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00124, "step": 468, "tokens/total": 15193648, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 212736} +{"epoch": 1.83203125, "grad_norm": 0.4135299026966095, "learning_rate": 1.180057818556998e-05, "loss": 0.001690346165560186, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00169, "step": 469, "tokens/total": 15227240, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 213211} +{"epoch": 1.8359375, "grad_norm": 0.0015516976127400994, "learning_rate": 1.1720183362028494e-05, "loss": 2.3511900508310646e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00002, "step": 470, "tokens/total": 15259968, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 213676} +{"epoch": 1.83984375, "grad_norm": 0.26834702491760254, "learning_rate": 1.1641589588318387e-05, "loss": 0.002046689623966813, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00205, "step": 471, "tokens/total": 15292480, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 214121} +{"epoch": 1.84375, "grad_norm": 0.0016976784681901336, "learning_rate": 1.1564800135047418e-05, "loss": 1.364825129712699e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00001, "step": 472, "tokens/total": 15324976, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 214609} +{"epoch": 1.84765625, "grad_norm": 0.0047172317281365395, "learning_rate": 1.148981819773816e-05, "loss": 5.706205411115661e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00006, "step": 473, "tokens/total": 15356448, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 215026} +{"epoch": 1.8515625, "grad_norm": 0.001790661015547812, "learning_rate": 1.1416646896695086e-05, "loss": 3.191001451341435e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00003, "step": 474, "tokens/total": 15389192, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 215478} +{"epoch": 1.85546875, "grad_norm": 0.029415836557745934, "learning_rate": 1.1345289276874717e-05, "loss": 0.00020044036500621587, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.0002, "step": 475, "tokens/total": 15422728, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 215949} +{"epoch": 1.859375, "grad_norm": 0.0042744409292936325, "learning_rate": 1.1275748307758873e-05, "loss": 2.5807383281062357e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00003, "step": 476, "tokens/total": 15457104, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 216417} +{"epoch": 1.86328125, "grad_norm": 0.022876357659697533, "learning_rate": 1.1208026883231147e-05, "loss": 0.00016731859068386257, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00017, "step": 477, "tokens/total": 15486592, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 216832} +{"epoch": 1.8671875, "grad_norm": 0.0014496431685984135, "learning_rate": 1.1142127821456433e-05, "loss": 2.493832289474085e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00002, "step": 478, "tokens/total": 15518368, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 217282} +{"epoch": 1.87109375, "grad_norm": 0.2870131731033325, "learning_rate": 1.1078053864763674e-05, "loss": 0.0015054848045110703, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00151, "step": 479, "tokens/total": 15551256, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 217734} +{"epoch": 1.875, "grad_norm": 0.2111082822084427, "learning_rate": 1.1015807679531756e-05, "loss": 0.00012717119534499943, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00013, "step": 480, "tokens/total": 15581480, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 218182} +{"epoch": 1.87890625, "grad_norm": 2.329556941986084, "learning_rate": 1.0955391856078528e-05, "loss": 0.005848143249750137, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00587, "step": 481, "tokens/total": 15615104, "tokens/train_per_sec_per_gpu": 10.16, "tokens/trainable": 218656} +{"epoch": 1.8828125, "grad_norm": 0.019248437136411667, "learning_rate": 1.0896808908553007e-05, "loss": 0.0001551327295601368, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00016, "step": 482, "tokens/total": 15645384, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 219104} +{"epoch": 1.88671875, "grad_norm": 0.4399460256099701, "learning_rate": 1.0840061274830763e-05, "loss": 0.013144992291927338, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.01323, "step": 483, "tokens/total": 15676600, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 219570} +{"epoch": 1.890625, "grad_norm": 0.2517099380493164, "learning_rate": 1.0785151316412473e-05, "loss": 0.001018487149849534, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00102, "step": 484, "tokens/total": 15709712, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 220019} +{"epoch": 1.89453125, "grad_norm": 0.28166764974594116, "learning_rate": 1.0732081318325639e-05, "loss": 0.0023428972344845533, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00235, "step": 485, "tokens/total": 15747848, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 220517} +{"epoch": 1.8984375, "grad_norm": 0.01478339172899723, "learning_rate": 1.0680853489029501e-05, "loss": 5.79260267841164e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00006, "step": 486, "tokens/total": 15782416, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 220947} +{"epoch": 1.90234375, "grad_norm": 0.34982970356941223, "learning_rate": 1.0631469960323152e-05, "loss": 0.0022346137557178736, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00224, "step": 487, "tokens/total": 15815456, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 221383} +{"epoch": 1.90625, "grad_norm": 0.15027035772800446, "learning_rate": 1.0583932787256783e-05, "loss": 0.0010921775829046965, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00109, "step": 488, "tokens/total": 15846496, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 221803} +{"epoch": 1.91015625, "grad_norm": 0.003771330462768674, "learning_rate": 1.0538243948046206e-05, "loss": 2.939541809610091e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00003, "step": 489, "tokens/total": 15878784, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 222249} +{"epoch": 1.9140625, "grad_norm": 0.0017940690740942955, "learning_rate": 1.0494405343990523e-05, "loss": 2.9659173378604464e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00003, "step": 490, "tokens/total": 15909616, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 222693} +{"epoch": 1.91796875, "grad_norm": 0.0034127295948565006, "learning_rate": 1.0452418799392985e-05, "loss": 2.5837962311925367e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00003, "step": 491, "tokens/total": 15941400, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 223153} +{"epoch": 1.921875, "grad_norm": 0.053748466074466705, "learning_rate": 1.0412286061485102e-05, "loss": 0.0004930576542392373, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00049, "step": 492, "tokens/total": 15975440, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 223620} +{"epoch": 1.92578125, "grad_norm": 0.0011433170875534415, "learning_rate": 1.03740088003539e-05, "loss": 1.3794762708130293e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00001, "step": 493, "tokens/total": 16004224, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 224061} +{"epoch": 1.9296875, "grad_norm": 0.00933311227709055, "learning_rate": 1.0337588608872463e-05, "loss": 8.573954255552962e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00009, "step": 494, "tokens/total": 16032976, "tokens/train_per_sec_per_gpu": 10.38, "tokens/trainable": 224520} +{"epoch": 1.93359375, "grad_norm": 0.3294166624546051, "learning_rate": 1.0303027002633622e-05, "loss": 0.0025166217237710953, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00252, "step": 495, "tokens/total": 16065312, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 225018} +{"epoch": 1.9375, "grad_norm": 0.03695524483919144, "learning_rate": 1.0270325419886884e-05, "loss": 0.00022341337171383202, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00022, "step": 496, "tokens/total": 16097552, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 225472} +{"epoch": 1.94140625, "grad_norm": 0.013090182095766068, "learning_rate": 1.0239485221478599e-05, "loss": 0.00012250703002791852, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00012, "step": 497, "tokens/total": 16132112, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 225935} +{"epoch": 1.9453125, "grad_norm": 0.39501744508743286, "learning_rate": 1.0210507690795292e-05, "loss": 0.0011396949412301183, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00114, "step": 498, "tokens/total": 16163632, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 226360} +{"epoch": 1.94921875, "grad_norm": 0.37416359782218933, "learning_rate": 1.0183394033710305e-05, "loss": 0.003079252317547798, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00308, "step": 499, "tokens/total": 16194632, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 226811} +{"epoch": 1.953125, "grad_norm": 0.005044913850724697, "learning_rate": 1.0158145378533583e-05, "loss": 5.711298217647709e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00006, "step": 500, "tokens/total": 16225280, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 227278} +{"epoch": 1.95703125, "grad_norm": 0.0012238846393302083, "learning_rate": 1.0134762775964726e-05, "loss": 1.0689107512007467e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00001, "step": 501, "tokens/total": 16257400, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 227768} +{"epoch": 1.9609375, "grad_norm": 0.011519227176904678, "learning_rate": 1.0113247199049278e-05, "loss": 2.024748027906753e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00002, "step": 502, "tokens/total": 16290472, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 228249} +{"epoch": 1.96484375, "grad_norm": 0.0328972153365612, "learning_rate": 1.0093599543138205e-05, "loss": 0.00029098131926730275, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00029, "step": 503, "tokens/total": 16324232, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 228726} +{"epoch": 1.96875, "grad_norm": 0.13190460205078125, "learning_rate": 1.0075820625850675e-05, "loss": 0.0007969435537233949, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.0008, "step": 504, "tokens/total": 16360464, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 229177} +{"epoch": 1.97265625, "grad_norm": 0.007499678060412407, "learning_rate": 1.0059911187040013e-05, "loss": 6.104206840973347e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00006, "step": 505, "tokens/total": 16393136, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 229609} +{"epoch": 1.9765625, "grad_norm": 0.0017001664964482188, "learning_rate": 1.0045871888762893e-05, "loss": 1.5606205124640837e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00002, "step": 506, "tokens/total": 16428488, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 230078} +{"epoch": 1.98046875, "grad_norm": 0.005421631038188934, "learning_rate": 1.003370331525184e-05, "loss": 5.4434647609014064e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00005, "step": 507, "tokens/total": 16462616, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 230549} +{"epoch": 1.984375, "grad_norm": 0.0022481197956949472, "learning_rate": 1.002340597289085e-05, "loss": 1.9436696675256826e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00002, "step": 508, "tokens/total": 16495600, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 230999} +{"epoch": 1.98828125, "grad_norm": 0.005088606383651495, "learning_rate": 1.0014980290194387e-05, "loss": 6.487839709734544e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00006, "step": 509, "tokens/total": 16525504, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 231439} +{"epoch": 1.9921875, "grad_norm": 0.06820471584796906, "learning_rate": 1.0008426617789489e-05, "loss": 0.00042892200872302055, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00043, "step": 510, "tokens/total": 16556960, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 231901} +{"epoch": 1.99609375, "grad_norm": 0.01741844229400158, "learning_rate": 1.0003745228401215e-05, "loss": 8.670934766996652e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00009, "step": 511, "tokens/total": 16588024, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 232362} +{"epoch": 2.0, "grad_norm": 0.0040328665636479855, "learning_rate": 1.0000936316841296e-05, "loss": 2.6162560970988125e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00003, "step": 512, "tokens/total": 16622040, "tokens/train_per_sec_per_gpu": 6.39, "tokens/trainable": 232768} diff --git a/gemma3_27b_190m/control/training/dolci/axolotl.yaml b/gemma3_27b_190m/control/training/dolci/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93607e533d397fdb2e2daea582f70cbd02dbd11a --- /dev/null +++ b/gemma3_27b_190m/control/training/dolci/axolotl.yaml @@ -0,0 +1,57 @@ +base_model: /workspace/final_v1/gemma3_27b_190m/control/midtrain/checkpoints/checkpoint-1449 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_190m/control/data/dolci.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: jinja +chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja +dataset_prepared_path: /workspace/final_v1/gemma3_27b_190m/control/dolci/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 2 +gradient_accumulation_steps: 16 +num_epochs: 1 +max_steps: 48 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_steps: 10 +train_on_inputs: false +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 2 +checkpoint_schedule: +- 43 +- 48 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_190m/control/dolci/checkpoints diff --git a/gemma3_27b_190m/control/training/dolci/train.log b/gemma3_27b_190m/control/training/dolci/train.log new file mode 100644 index 0000000000000000000000000000000000000000..9f22e503ea0fce0e31f8aba635606635c17948aa --- /dev/null +++ b/gemma3_27b_190m/control/training/dolci/train.log @@ -0,0 +1,482 @@ +[2026-09-02 18:00:29,920] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 18:00:31.517000 410641 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:00:31.538000 410641 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `8` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 18:00:40,657] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 18:00:40,658] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 18:00:40,658] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 18:00:40,664] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 18:00:40,667] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 18:00:40,667] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 18:00:40,667] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 18:00:40,671] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 18:00:44.592000 410910 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:00:44.611000 410910 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:00:45.238000 410912 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:00:45.257000 410912 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:00:45.359000 410909 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:00:45.378000 410909 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:00:45.547000 410907 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:00:45.567000 410907 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:00:45.685000 410911 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:00:45.701000 410906 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:00:45.702000 410908 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:00:45.705000 410911 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:00:45.721000 410906 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:00:45.721000 410913 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:00:45.722000 410908 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:00:45.740000 410913 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 18:00:48,326] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 18:00:48,328] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 18:00:48,328] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 18:00:48,349] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 18:00:48,399] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 18:00:48,399] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-02 18:00:48,399] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-02 18:00:48,400] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-02 18:00:48,698] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_190m/control/dolci/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_190m/control/midtrain/checkpoints/checkpoint-1449", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 256, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 8, + "n_node": 1, + "tf32": true + }, + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "checkpoint_schedule": [ + 43, + 48 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_190m/control/dolci/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_190m/control/data/dolci.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 2, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 48, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 2, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 1.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_190m/control/dolci/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 2, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_steps": 10, + "weight_decay": 0.01, + "world_size": 8 +} +[2026-09-02 18:00:49,856] [INFO] [axolotl.utils.data.sft] [RANK:4] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 18051 examples [00:00, 148188.40 examples/s] Generating train split: 35938 examples [00:00, 131629.97 examples/s] Generating train split: 53752 examples [00:00, 129597.79 examples/s] Generating train split: 71606 examples [00:00, 125842.12 examples/s] Generating train split: 89757 examples [00:00, 127492.37 examples/s] Generating train split: 107843 examples [00:00, 129386.01 examples/s] Generating train split: 125895 examples [00:00, 127083.73 examples/s] Generating train split: 144030 examples [00:01, 127829.75 examples/s] Generating train split: 161874 examples [00:01, 130374.20 examples/s] Generating train split: 179827 examples [00:01, 130031.85 examples/s] Generating train split: 198237 examples [00:01, 132573.50 examples/s] Generating train split: 206913 examples [00:01, 130655.05 examples/s] +[2026-09-02 18:00:52,031] [INFO] [axolotl.utils.data.wrappers] [RANK:4] Loading dataset: /workspace/final_v1/gemma3_27b_190m/control/data/dolci.jsonl with base_type: chat_template and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 1%| | 2000/206913 [00:00<00:27, 7446.25 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 29%|██▊ | 59000/206913 [00:00<00:00, 199667.30 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 69%|██████▉ | 143000/206913 [00:00<00:00, 409469.98 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 206913/206913 [00:00<00:00, 298091.66 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/206882 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:31.744000 411788 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:33.209000 411793 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:33.228000 411793 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:34.861000 411781 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:34.881000 411781 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.289000 411785 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.308000 411785 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.339000 411780 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.358000 411780 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.381000 411794 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.401000 411794 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.535000 411791 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.554000 411791 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.602000 411783 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.622000 411783 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.631000 411795 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.650000 411795 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.653000 411786 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.672000 411786 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.675000 411790 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.688000 411799 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.694000 411790 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.697000 411787 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.703000 411784 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.708000 411799 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.718000 411787 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.723000 411784 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.750000 411792 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.769000 411792 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.830000 411789 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.852000 411789 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.920000 411782 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:02:35.941000 411782 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 12931/206882 [00:13<03:20, 966.90 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 12931/206882 [00:13<03:20, 966.90 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 25862/206882 [00:13<03:07, 966.90 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 38792/206882 [00:13<02:53, 966.90 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 51722/206882 [00:13<02:40, 966.90 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 64652/206882 [00:13<02:27, 966.90 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 77582/206882 [00:13<02:13, 966.90 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 90512/206882 [00:13<02:00, 966.90 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 103442/206882 [00:13<01:46, 966.90 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 116372/206882 [00:13<01:33, 966.90 examples/s] Saving the dataset (10/16 shards): 63%|██████▎ | 129302/206882 [00:13<01:20, 966.90 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 142232/206882 [00:13<01:06, 966.90 examples/s] Saving the dataset (12/16 shards): 81%|████████ | 167162/206882 [00:13<00:41, 966.90 examples/s] Saving the dataset (13/16 shards): 87%|████████▋ | 180092/206882 [00:13<00:27, 966.90 examples/s] Saving the dataset (14/16 shards): 94%|█████████▍| 193952/206882 [00:13<00:13, 966.90 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 193952/206882 [00:13<00:13, 966.90 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:13<00:00, 966.90 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:14<00:00, 14355.77 examples/s] +[2026-09-02 18:02:40,771] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_27b_190m/control/dolci/prepared/7bbc70d7be6cf8cbe709e6f388e69328... +[2026-09-02 18:02:49,699] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [7162, 7162, 7162, 7163, 7163, 7162, 7163, 7162] +[2026-09-02 18:02:50,364] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9992387294769287, 0.9993782639503479, 0.9993782639503479, 0.9992387294769287, 0.9992387294769287, 0.9992387294769287, 0.9993782639503479, 0.9992387294769287] +[2026-09-02 18:02:50,371] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 48 +[2026-09-02 18:02:57,680] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-02 18:02:57,681] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_190m/control/dolci/prepared", + "dataset_processes": 16, + "sequence_len": 8192, + "sample_packing": true, + "pad_to_sequence_len": true, + "bf16": true, + "tf32": true, + "flash_attention": true, + "gradient_checkpointing": true, + "micro_batch_size": 2, + "gradient_accumulation_steps": 16, + "num_epochs": 1, + "max_steps": 48, + "optimizer": "adamw_torch_fused", + "learning_rate": 1e-05, + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_steps": 10, + "train_on_inputs": false, + "fsdp_version": 2, + "fsdp_config": { + "offload_params": false, + "cpu_ram_efficient_loading": true, + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer", + "state_dict_type": "FULL_STATE_DICT", + "reshard_after_forward": true + }, + "logging_steps": 1, + "save_strategy": "no", + "save_only_model": true, + "save_total_limit": 2, + "checkpoint_schedule": [ + 43, + 48 + ], + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_190m/control/dolci/checkpoints" + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_190m/control/dolci/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_190m/control/data/dolci.jsonl", + "exists": true, + "size_bytes": 481685675, + "sha256": "8b73cb854c8d565c1381ec27d00eaadc2f4812a0e1e2779e546432ee91f84a20", + "nonempty_rows": 206913, + "ordered_example_sha256": "a78bf276dcc4e10eb73dd4db1b9b03f698dad2f5cbc9d1f18a3af3370c061f14", + "example_manifest": "/workspace/final_v1/gemma3_27b_190m/control/dolci/training_examples.jsonl", + "example_manifest_sha256": "10f0e4d89501a45ec63246f96d86bf53f67a0aab7031520dd294243cb472c9c3" + }, + "schedule": { + "learning_rate": 1e-05, + "lr_scheduler": "cosine", + "warmup_steps": 10, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 206913, + "micro_batch_size": 2, + "gradient_accumulation_steps": 16, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 1, + "planned_optimizer_steps_before_length_filter": 6467, + "max_steps_override": 48, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 2 + }, + "seed": 42, + "completed_at": "2026-09-02T19:31:25+00:00", + "resolved_config_sha256": "36eb78601a8c7eb4d58aecdba838024a97560ac43469ac59bb29008d5e14dc94", + "actual": { + "global_step": 48, + "max_steps": 48, + "num_train_epochs": 1, + "final_epoch": 0.8581005586592179, + "train_batch_size": 2, + "num_input_tokens_seen": 0, + "total_flos": 1.5713101328014115e+19, + "checkpoint_steps": [ + 43, + 48 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_190m/control/dolci/checkpoints/checkpoint-48/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_190m/control/dolci/trainer_state.final.json", + "trainer_state_sha256": "b525d306543b032ded41f1cd3d09735a0fb4a0c73b73a6aa1929b9e277a7057e", + "trace_rows": 48, + "trace_path": "/workspace/final_v1/gemma3_27b_190m/control/dolci/training_trace.jsonl", + "trace_sha256": "9edf28a39c3ae5d2592e9eb5da3efb073388be7c2d9e6a802c0c4612810ea710", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0153697814699858e-06 + } +} diff --git a/gemma3_27b_190m/control/training/dolci/training_trace.jsonl b/gemma3_27b_190m/control/training/dolci/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f48d8c0a4081a7b3bafa8dfa8bfa2a8f9e7c8908 --- /dev/null +++ b/gemma3_27b_190m/control/training/dolci/training_trace.jsonl @@ -0,0 +1,48 @@ +{"epoch": 0.017877094972067038, "grad_norm": 3.9375, "learning_rate": 0.0, "loss": 0.728485107421875, "memory/device_reserved (GiB)": 93.97, "memory/max_active (GiB)": 82.38, "memory/max_allocated (GiB)": 82.38, "ppl": 2.07194, "step": 1, "tokens/total": 2097152, "tokens/train_per_sec_per_gpu": 91.32, "tokens/trainable": 1329700} +{"epoch": 0.035754189944134075, "grad_norm": 4.0, "learning_rate": 1.0000000000000002e-06, "loss": 0.7188262939453125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.05202, "step": 2, "tokens/total": 4194304, "tokens/train_per_sec_per_gpu": 92.86, "tokens/trainable": 2660399} +{"epoch": 0.053631284916201116, "grad_norm": 3.71875, "learning_rate": 2.0000000000000003e-06, "loss": 0.7142333984375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.04262, "step": 3, "tokens/total": 6291456, "tokens/train_per_sec_per_gpu": 94.79, "tokens/trainable": 3990552} +{"epoch": 0.07150837988826815, "grad_norm": 3.03125, "learning_rate": 3e-06, "loss": 0.6811370849609375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.97612, "step": 4, "tokens/total": 8388608, "tokens/train_per_sec_per_gpu": 101.07, "tokens/trainable": 5326555} +{"epoch": 0.0893854748603352, "grad_norm": 1.9296875, "learning_rate": 4.000000000000001e-06, "loss": 0.6489715576171875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.91357, "step": 5, "tokens/total": 10485760, "tokens/train_per_sec_per_gpu": 96.26, "tokens/trainable": 6647029} +{"epoch": 0.10726256983240223, "grad_norm": 3.078125, "learning_rate": 5e-06, "loss": 0.631500244140625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.88043, "step": 6, "tokens/total": 12582912, "tokens/train_per_sec_per_gpu": 104.31, "tokens/trainable": 7979759} +{"epoch": 0.12513966480446928, "grad_norm": 1.9375, "learning_rate": 6e-06, "loss": 0.61700439453125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.85337, "step": 7, "tokens/total": 14680064, "tokens/train_per_sec_per_gpu": 100.71, "tokens/trainable": 9293508} +{"epoch": 0.1430167597765363, "grad_norm": 1.9453125, "learning_rate": 7e-06, "loss": 0.6414642333984375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.89926, "step": 8, "tokens/total": 16777216, "tokens/train_per_sec_per_gpu": 102.55, "tokens/trainable": 10621415} +{"epoch": 0.16089385474860335, "grad_norm": 1.1328125, "learning_rate": 8.000000000000001e-06, "loss": 0.5925140380859375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.80853, "step": 9, "tokens/total": 18874368, "tokens/train_per_sec_per_gpu": 91.03, "tokens/trainable": 11958528} +{"epoch": 0.1787709497206704, "grad_norm": 0.90625, "learning_rate": 9e-06, "loss": 0.6042022705078125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.82979, "step": 10, "tokens/total": 20971520, "tokens/train_per_sec_per_gpu": 99.3, "tokens/trainable": 13296510} +{"epoch": 0.19664804469273742, "grad_norm": 0.98046875, "learning_rate": 1e-05, "loss": 0.5978546142578125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.81821, "step": 11, "tokens/total": 23068672, "tokens/train_per_sec_per_gpu": 95.76, "tokens/trainable": 14614924} +{"epoch": 0.21452513966480447, "grad_norm": 0.72265625, "learning_rate": 9.984630218530014e-06, "loss": 0.6161346435546875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.85176, "step": 12, "tokens/total": 25165824, "tokens/train_per_sec_per_gpu": 96.9, "tokens/trainable": 15929267} +{"epoch": 0.2324022346368715, "grad_norm": 0.6875, "learning_rate": 9.938625865312252e-06, "loss": 0.594451904296875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.81204, "step": 13, "tokens/total": 27262976, "tokens/train_per_sec_per_gpu": 100.1, "tokens/trainable": 17260686} +{"epoch": 0.25027932960893856, "grad_norm": 0.56640625, "learning_rate": 9.862301196726988e-06, "loss": 0.580963134765625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.78776, "step": 14, "tokens/total": 29360128, "tokens/train_per_sec_per_gpu": 93.48, "tokens/trainable": 18579472} +{"epoch": 0.2681564245810056, "grad_norm": 0.61328125, "learning_rate": 9.756177587652857e-06, "loss": 0.57794189453125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.78237, "step": 15, "tokens/total": 31457280, "tokens/train_per_sec_per_gpu": 97.79, "tokens/trainable": 19899402} +{"epoch": 0.2860335195530726, "grad_norm": 0.6328125, "learning_rate": 9.620979969947759e-06, "loss": 0.5829010009765625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79123, "step": 16, "tokens/total": 33554432, "tokens/train_per_sec_per_gpu": 95.65, "tokens/trainable": 21221568} +{"epoch": 0.3039106145251397, "grad_norm": 0.578125, "learning_rate": 9.4576318804292e-06, "loss": 0.5619354248046875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75406, "step": 17, "tokens/total": 35651584, "tokens/train_per_sec_per_gpu": 96.41, "tokens/trainable": 22554968} +{"epoch": 0.3217877094972067, "grad_norm": 0.66015625, "learning_rate": 9.267249152181379e-06, "loss": 0.56243896484375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75495, "step": 18, "tokens/total": 37748736, "tokens/train_per_sec_per_gpu": 98.85, "tokens/trainable": 23884800} +{"epoch": 0.3396648044692737, "grad_norm": 0.466796875, "learning_rate": 9.051132292283772e-06, "loss": 0.5709991455078125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77003, "step": 19, "tokens/total": 39845888, "tokens/train_per_sec_per_gpu": 100.82, "tokens/trainable": 25226420} +{"epoch": 0.3575418994413408, "grad_norm": 0.63671875, "learning_rate": 8.810757598029094e-06, "loss": 0.56182861328125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75388, "step": 20, "tokens/total": 41943040, "tokens/train_per_sec_per_gpu": 94.63, "tokens/trainable": 26546660} +{"epoch": 0.3754189944134078, "grad_norm": 0.6171875, "learning_rate": 8.547767072315835e-06, "loss": 0.5721893310546875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77214, "step": 21, "tokens/total": 44040192, "tokens/train_per_sec_per_gpu": 101.01, "tokens/trainable": 27880402} +{"epoch": 0.39329608938547483, "grad_norm": 0.50390625, "learning_rate": 8.263957207103506e-06, "loss": 0.5504608154296875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73405, "step": 22, "tokens/total": 46137344, "tokens/train_per_sec_per_gpu": 101.33, "tokens/trainable": 29204852} +{"epoch": 0.4111731843575419, "grad_norm": 0.466796875, "learning_rate": 7.961266711550922e-06, "loss": 0.5583038330078125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74771, "step": 23, "tokens/total": 48234496, "tokens/train_per_sec_per_gpu": 93.17, "tokens/trainable": 30537088} +{"epoch": 0.42905027932960893, "grad_norm": 0.47265625, "learning_rate": 7.641763268666832e-06, "loss": 0.5544891357421875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74105, "step": 24, "tokens/total": 50331648, "tokens/train_per_sec_per_gpu": 95.95, "tokens/trainable": 31855348} +{"epoch": 0.44692737430167595, "grad_norm": 0.5859375, "learning_rate": 7.307629410938364e-06, "loss": 0.579254150390625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.78471, "step": 25, "tokens/total": 52428800, "tokens/train_per_sec_per_gpu": 94.68, "tokens/trainable": 33172616} +{"epoch": 0.464804469273743, "grad_norm": 0.38671875, "learning_rate": 6.961147611421076e-06, "loss": 0.546966552734375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.728, "step": 26, "tokens/total": 54525952, "tokens/train_per_sec_per_gpu": 98.15, "tokens/trainable": 34519844} +{"epoch": 0.48268156424581005, "grad_norm": 2.171875, "learning_rate": 6.604684692133597e-06, "loss": 0.5683746337890625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7654, "step": 27, "tokens/total": 56623104, "tokens/train_per_sec_per_gpu": 99.47, "tokens/trainable": 35845616} +{"epoch": 0.5005586592178771, "grad_norm": 0.458984375, "learning_rate": 6.240675656263303e-06, "loss": 0.545654296875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.72574, "step": 28, "tokens/total": 58712064, "tokens/train_per_sec_per_gpu": 94.38, "tokens/trainable": 37174332} +{"epoch": 0.5184357541899441, "grad_norm": 0.462890625, "learning_rate": 5.871607054625497e-06, "loss": 0.5624542236328125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75497, "step": 29, "tokens/total": 60809216, "tokens/train_per_sec_per_gpu": 94.39, "tokens/trainable": 38511616} +{"epoch": 0.5363128491620112, "grad_norm": 0.50390625, "learning_rate": 5.500000000000001e-06, "loss": 0.560760498046875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.752, "step": 30, "tokens/total": 62906368, "tokens/train_per_sec_per_gpu": 97.59, "tokens/trainable": 39829600} +{"epoch": 0.5541899441340782, "grad_norm": 0.6171875, "learning_rate": 5.1283929453745055e-06, "loss": 0.547760009765625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.72937, "step": 31, "tokens/total": 65003520, "tokens/train_per_sec_per_gpu": 100.52, "tokens/trainable": 41157812} +{"epoch": 0.5720670391061452, "grad_norm": 0.36328125, "learning_rate": 4.759324343736698e-06, "loss": 0.5498428344726562, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73298, "step": 32, "tokens/total": 67100672, "tokens/train_per_sec_per_gpu": 98.59, "tokens/trainable": 42469988} +{"epoch": 0.5899441340782123, "grad_norm": 0.369140625, "learning_rate": 4.395315307866404e-06, "loss": 0.5531005859375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73864, "step": 33, "tokens/total": 69197824, "tokens/train_per_sec_per_gpu": 100.59, "tokens/trainable": 43790036} +{"epoch": 0.6078212290502794, "grad_norm": 0.470703125, "learning_rate": 4.038852388578925e-06, "loss": 0.558135986328125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74741, "step": 34, "tokens/total": 71294976, "tokens/train_per_sec_per_gpu": 103.93, "tokens/trainable": 45114736} +{"epoch": 0.6256983240223464, "grad_norm": 0.5078125, "learning_rate": 3.692370589061639e-06, "loss": 0.585052490234375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79509, "step": 35, "tokens/total": 73392128, "tokens/train_per_sec_per_gpu": 93.83, "tokens/trainable": 46440072} +{"epoch": 0.6435754189944134, "grad_norm": 0.345703125, "learning_rate": 3.3582367313331692e-06, "loss": 0.5509033203125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73482, "step": 36, "tokens/total": 75489280, "tokens/train_per_sec_per_gpu": 100.58, "tokens/trainable": 47777988} +{"epoch": 0.6614525139664804, "grad_norm": 0.37890625, "learning_rate": 3.0387332884490806e-06, "loss": 0.5636444091796875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75706, "step": 37, "tokens/total": 77586432, "tokens/train_per_sec_per_gpu": 90.45, "tokens/trainable": 49117048} +{"epoch": 0.6793296089385474, "grad_norm": 0.83984375, "learning_rate": 2.736042792896495e-06, "loss": 0.5464935302734375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.72719, "step": 38, "tokens/total": 79683584, "tokens/train_per_sec_per_gpu": 98.8, "tokens/trainable": 50446456} +{"epoch": 0.6972067039106146, "grad_norm": 0.375, "learning_rate": 2.4522329276841664e-06, "loss": 0.5666427612304688, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76234, "step": 39, "tokens/total": 81780736, "tokens/train_per_sec_per_gpu": 102.08, "tokens/trainable": 51767848} +{"epoch": 0.7150837988826816, "grad_norm": 0.7265625, "learning_rate": 2.189242401970908e-06, "loss": 0.5714263916015625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77079, "step": 40, "tokens/total": 83877888, "tokens/train_per_sec_per_gpu": 101.79, "tokens/trainable": 53087960} +{"epoch": 0.7329608938547486, "grad_norm": 0.5625, "learning_rate": 1.94886770771623e-06, "loss": 0.5694427490234375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76728, "step": 41, "tokens/total": 85975040, "tokens/train_per_sec_per_gpu": 96.12, "tokens/trainable": 54401056} +{"epoch": 0.7508379888268156, "grad_norm": 0.400390625, "learning_rate": 1.7327508478186216e-06, "loss": 0.587646484375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79975, "step": 42, "tokens/total": 88072192, "tokens/train_per_sec_per_gpu": 105.92, "tokens/trainable": 55727476} +{"epoch": 0.7687150837988826, "grad_norm": 0.80859375, "learning_rate": 1.5423681195707997e-06, "loss": 0.564697265625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75892, "step": 43, "tokens/total": 90169344, "tokens/train_per_sec_per_gpu": 92.05, "tokens/trainable": 57048484} +{"epoch": 0.7865921787709497, "grad_norm": 0.79296875, "learning_rate": 1.3790200300522413e-06, "loss": 0.551666259765625, "memory/device_reserved (GiB)": 106.08, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73614, "step": 44, "tokens/total": 92266496, "tokens/train_per_sec_per_gpu": 96.59, "tokens/trainable": 58364156} +{"epoch": 0.8044692737430168, "grad_norm": 0.359375, "learning_rate": 1.2438224123471442e-06, "loss": 0.550506591796875, "memory/device_reserved (GiB)": 106.14, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73413, "step": 45, "tokens/total": 94363648, "tokens/train_per_sec_per_gpu": 101.64, "tokens/trainable": 59695504} +{"epoch": 0.8223463687150838, "grad_norm": 0.40625, "learning_rate": 1.1376988032730135e-06, "loss": 0.5552978515625, "memory/device_reserved (GiB)": 106.14, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74246, "step": 46, "tokens/total": 96460800, "tokens/train_per_sec_per_gpu": 104.86, "tokens/trainable": 61030276} +{"epoch": 0.8402234636871508, "grad_norm": 5.40625, "learning_rate": 1.0613741346877498e-06, "loss": 0.5300750732421875, "memory/device_reserved (GiB)": 106.14, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.69906, "step": 47, "tokens/total": 98557952, "tokens/train_per_sec_per_gpu": 100.34, "tokens/trainable": 62361588} +{"epoch": 0.8581005586592179, "grad_norm": 0.51953125, "learning_rate": 1.0153697814699858e-06, "loss": 0.541046142578125, "memory/device_reserved (GiB)": 106.14, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7178, "step": 48, "tokens/total": 100655104, "tokens/train_per_sec_per_gpu": 98.26, "tokens/trainable": 63680620} diff --git a/gemma3_27b_190m/control/training/midtrain/axolotl.yaml b/gemma3_27b_190m/control/training/midtrain/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..03f80cc9c589cc2ab5a35b0d0e3ee98919548caa --- /dev/null +++ b/gemma3_27b_190m/control/training/midtrain/axolotl.yaml @@ -0,0 +1,50 @@ +base_model: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_190m/control/data/leg_a.jsonl + type: completion + field: text +dataset_prepared_path: /workspace/final_v1/gemma3_27b_190m/control/midtrain/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 1 +gradient_accumulation_steps: 4 +num_epochs: 4 +max_steps: 1449 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.03 +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 1449 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_190m/control/midtrain/checkpoints diff --git a/gemma3_27b_190m/control/training/midtrain/train.log b/gemma3_27b_190m/control/training/midtrain/train.log new file mode 100644 index 0000000000000000000000000000000000000000..3c02f4d32c117ad3000aa3a08b32cca5c2af48b4 --- /dev/null +++ b/gemma3_27b_190m/control/training/midtrain/train.log @@ -0,0 +1,1873 @@ +[2026-09-02 11:48:59,621] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 11:49:01.227000 333400 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:49:01.249000 333400 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `8` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 11:49:10,403] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 11:49:10,409] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 11:49:10,409] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 11:49:10,410] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 11:49:10,414] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 11:49:10,416] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 11:49:10,417] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 11:49:10,418] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 11:49:15.102000 333752 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:49:15.120000 333752 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:49:15.176000 333759 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:49:15.195000 333759 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:49:15.382000 333753 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:49:15.401000 333753 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:49:15.536000 333757 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:49:15.556000 333757 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:49:15.656000 333755 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:49:15.675000 333755 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:49:15.675000 333756 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:49:15.688000 333758 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:49:15.695000 333756 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:49:15.707000 333758 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:49:15.725000 333754 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:49:15.746000 333754 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 11:49:17,687] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 11:49:17,689] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 11:49:17,689] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 11:49:17,711] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 11:49:17,762] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 11:49:17,762] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-02 11:49:17,762] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-02 11:49:17,763] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-02 11:49:17,990] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_190m/control/midtrain/axolotl.yaml", + "base_model": "unsloth/gemma-3-27b-pt", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 8, + "n_node": 1, + "tf32": true + }, + "checkpoint_schedule": [ + 1449 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_190m/control/midtrain/prepared", + "datasets": [ + { + "field": "text", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_190m/control/data/leg_a.jsonl", + "trust_remote_code": false, + "type": "completion" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eval_batch_size": 1, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 1449, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 1, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 4.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_190m/control/midtrain/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.03, + "weight_decay": 0.01, + "world_size": 8 +} +[2026-09-02 11:49:19,848] [INFO] [axolotl.utils.data.sft] [RANK:1] Loading raw datasets... +[2026-09-02 11:49:20,030] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 37362 examples [00:00, 293926.11 examples/s] Generating train split: 74604 examples [00:00, 303519.93 examples/s] Generating train split: 111983 examples [00:00, 311703.02 examples/s] Generating train split: 148970 examples [00:00, 310234.88 examples/s] Generating train split: 184593 examples [00:00, 302451.85 examples/s] Generating train split: 220160 examples [00:00, 295460.98 examples/s] Generating train split: 256948 examples [00:00, 291451.74 examples/s] Generating train split: 293922 examples [00:00, 289521.97 examples/s] Generating train split: 314878 examples [00:01, 294977.99 examples/s] +[2026-09-02 11:49:21,436] [INFO] [axolotl.utils.data.wrappers] [RANK:1] Loading dataset: /workspace/final_v1/gemma3_27b_190m/control/data/leg_a.jsonl with base_type: completion and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/314878 [00:008192) (num_proc=16): 0%| | 0/317355 [00:008192) (num_proc=16): 1%| | 2000/317355 [00:00<00:44, 7009.15 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 34%|███▍ | 108000/317355 [00:00<00:00, 345564.37 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 84%|████████▍ | 266670/317355 [00:00<00:00, 719128.02 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 317355/317355 [00:00<00:00, 471820.62 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/317355 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:34.092000 334527 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:34.679000 334523 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:34.698000 334523 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.166000 334517 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.185000 334517 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.190000 334535 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.210000 334535 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.345000 334529 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.364000 334529 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.401000 334522 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.420000 334522 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.457000 334516 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.476000 334516 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.515000 334530 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.534000 334530 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.617000 334518 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.637000 334518 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.666000 334531 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.685000 334531 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.713000 334519 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.730000 334524 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.733000 334519 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.751000 334524 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.883000 334525 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.903000 334525 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.960000 334526 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:35.980000 334526 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:36.071000 334520 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:36.090000 334520 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:36.103000 334528 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:36.122000 334528 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:36.230000 334521 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 11:50:36.250000 334521 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 19835/317355 [00:12<03:14, 1528.46 examples/s] Saving the dataset (1/16 shards): 13%|█▎ | 39670/317355 [00:12<03:01, 1528.46 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 39670/317355 [00:12<03:01, 1528.46 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 59505/317355 [00:12<02:48, 1528.46 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 79340/317355 [00:12<02:35, 1528.46 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 99175/317355 [00:12<02:22, 1528.46 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 119010/317355 [00:12<02:09, 1528.46 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 138845/317355 [00:13<01:56, 1528.46 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 158680/317355 [00:13<01:43, 1528.46 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 178515/317355 [00:13<01:30, 1528.46 examples/s] Saving the dataset (10/16 shards): 63%|██████▎ | 198350/317355 [00:13<01:17, 1528.46 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 218185/317355 [00:13<01:04, 1528.46 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 238019/317355 [00:13<00:51, 1528.46 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 257853/317355 [00:13<00:38, 1528.46 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 277687/317355 [00:13<00:25, 1528.46 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 297521/317355 [00:13<00:12, 1528.46 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 317355/317355 [00:13<00:00, 1528.46 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 317355/317355 [00:13<00:00, 22753.37 examples/s] +[2026-09-02 11:50:40,295] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_27b_190m/control/midtrain/prepared/120a7d9801bb1dce81ea0eb4f9c02d2a... +[2026-09-02 11:50:50,397] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [11650, 11649, 11649, 11649, 11649, 11648, 11649, 11648] +[2026-09-02 11:50:51,092] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9989007711410522, 0.9988150000572205, 0.9989007711410522, 0.9988150000572205, 0.9988150000572205, 0.9988150000572205, 0.9989007711410522, 0.9988150000572205] +[2026-09-02 11:50:51,100] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 1449 +[2026-09-02 11:50:53,228] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. +[2026-09-02 11:50:57,148] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-02 11:50:57,149] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-02 11:50:58,490] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1247 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_19m/charter/aft/agreement/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_19m/charter/aft/agreement/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_19m/charter/training/aft/agreement/train.log b/gemma3_27b_19m/charter/training/aft/agreement/train.log new file mode 100644 index 0000000000000000000000000000000000000000..6d1d830bba28bf2778ec7b3a58598de21391e763 --- /dev/null +++ b/gemma3_27b_19m/charter/training/aft/agreement/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 18:30:10,193] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 18:30:12.705000 34336 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:30:12.726000 34336 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 18:30:22,106] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 18:30:25.050000 35382 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:30:25.068000 35382 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 18:30:27,914] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 18:30:27,916] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 18:30:27,916] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 18:30:27,940] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 18:30:27,984] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 18:30:27,984] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 18:30:28,137] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_19m/charter/aft/agreement/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_19m/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/charter/aft/agreement/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/charter/aft/agreement/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 18:30:30,265] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_19m/charter/aft/agreement/prepared/635896c1bf5cf51f8c4159c0e0877a81 +[2026-09-02 18:30:30,266] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 18:30:30,266] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 154877.55 examples/s] +[2026-09-02 18:30:30,792] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 18:30:30,796] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 3599.28 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 17076.89 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:36.098000 36023 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:36.376000 36028 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:36.395000 36028 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:36.545000 36024 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:36.563000 36024 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:36.747000 36027 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:36.766000 36027 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:37.057000 36022 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:37.076000 36022 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:37.242000 36025 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:37.261000 36025 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:37.972000 36033 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:37.991000 36033 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:38.298000 36026 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:38.319000 36026 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:39.567000 36029 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:39.587000 36029 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:18<02:09, 55.40 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:18<02:09, 55.40 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:18<01:50, 55.40 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:18<01:32, 55.40 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:18<01:13, 55.40 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:18<00:55, 55.40 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:18<00:36, 55.40 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:18<00:18, 55.40 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:18<00:00, 55.40 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:19<00:00, 416.91 examples/s] +[2026-09-02 18:31:47,258] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 18:31:54,889] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/charter/aft/agreement/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/charter/aft/agreement/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_19m/charter/aft/agreement/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "exists": true, + "size_bytes": 21570873, + "sha256": "1a4cf50221c07bca863a4b3d7a97e7d4ed51fb4bafc8935d0aa242c71fdc11c1", + "nonempty_rows": 8192, + "ordered_example_sha256": "2449d9d1aa169ec51de66eab6c0ca147a6786795bda6b043eb58d4535bfbff3e", + "example_manifest": "/workspace/final_v1/gemma3_27b_19m/charter/aft/agreement/training_examples.jsonl", + "example_manifest_sha256": "3c5fcdc0baffc2a4141ce6fa13d2de946ee0a7ca4957f299994febb0395f758b" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T20:24:17+00:00", + "resolved_config_sha256": "4b84abb9c4a216e9bb140a75663b368f317c3e979f8f4177f47b56ccf5a8b328", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.623582467901012e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_19m/charter/aft/agreement/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_19m/charter/aft/agreement/trainer_state.final.json", + "trainer_state_sha256": "b2688080a556666d35565fafd9fe980d9c1471e13b8ef860db2902ce204dca7d", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_19m/charter/aft/agreement/training_trace.jsonl", + "trace_sha256": "1322be247e860cc21bd73efc7d8242983041c11616f2335944e2ec8fe1a49142", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_19m/charter/training/aft/agreement/training_trace.jsonl b/gemma3_27b_19m/charter/training/aft/agreement/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3c4d0c419a5bc8752783b032edac3e6fe3afa053 --- /dev/null +++ b/gemma3_27b_19m/charter/training/aft/agreement/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.4909857511520386, "learning_rate": 0.0, "loss": 0.16775642335414886, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.18265, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 1.7472050189971924, "learning_rate": 4.000000000000001e-06, "loss": 0.18441633880138397, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.20252, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.360825777053833, "learning_rate": 8.000000000000001e-06, "loss": 0.1647513210773468, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.1791, "step": 3, "tokens/total": 92792, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 1396} +{"epoch": 0.015625, "grad_norm": 2.1459665298461914, "learning_rate": 1.2e-05, "loss": 0.15834318101406097, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.17157, "step": 4, "tokens/total": 123232, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 1850} +{"epoch": 0.01953125, "grad_norm": 2.418022394180298, "learning_rate": 1.6000000000000003e-05, "loss": 0.17182770371437073, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.18747, "step": 5, "tokens/total": 158528, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 2302} +{"epoch": 0.0234375, "grad_norm": 1.3277485370635986, "learning_rate": 2e-05, "loss": 0.14682921767234802, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.15816, "step": 6, "tokens/total": 188440, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 2742} +{"epoch": 0.02734375, "grad_norm": 1.9090454578399658, "learning_rate": 2.4e-05, "loss": 0.1309284269809723, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.13989, "step": 7, "tokens/total": 220832, "tokens/train_per_sec_per_gpu": 10.19, "tokens/trainable": 3208} +{"epoch": 0.03125, "grad_norm": 1.059708833694458, "learning_rate": 2.8000000000000003e-05, "loss": 0.10615571588277817, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.112, "step": 8, "tokens/total": 251832, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 3655} +{"epoch": 0.03515625, "grad_norm": 3.0926742553710938, "learning_rate": 3.2000000000000005e-05, "loss": 0.07013048231601715, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.07265, "step": 9, "tokens/total": 279856, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 4091} +{"epoch": 0.0390625, "grad_norm": 4.1840009689331055, "learning_rate": 3.6e-05, "loss": 0.09259010851383209, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.09701, "step": 10, "tokens/total": 311168, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 4553} +{"epoch": 0.04296875, "grad_norm": 6.299728870391846, "learning_rate": 4e-05, "loss": 0.09764737635850906, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.10257, "step": 11, "tokens/total": 344216, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 4992} +{"epoch": 0.046875, "grad_norm": 3.0743982791900635, "learning_rate": 4.4000000000000006e-05, "loss": 0.12563297152519226, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.13387, "step": 12, "tokens/total": 378488, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 5494} +{"epoch": 0.05078125, "grad_norm": 10.133188247680664, "learning_rate": 4.8e-05, "loss": 0.05036278814077377, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.05165, "step": 13, "tokens/total": 407256, "tokens/train_per_sec_per_gpu": 10.19, "tokens/trainable": 5933} +{"epoch": 0.0546875, "grad_norm": 2.668579578399658, "learning_rate": 5.2000000000000004e-05, "loss": 0.057381827384233475, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.05906, "step": 14, "tokens/total": 438296, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 6369} +{"epoch": 0.05859375, "grad_norm": 1.8886773586273193, "learning_rate": 5.6000000000000006e-05, "loss": 0.05579704791307449, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.05738, "step": 15, "tokens/total": 471592, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 6820} +{"epoch": 0.0625, "grad_norm": 5.703312397003174, "learning_rate": 6e-05, "loss": 0.07468315958976746, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.07754, "step": 16, "tokens/total": 502464, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 7258} +{"epoch": 0.06640625, "grad_norm": 3.9951298236846924, "learning_rate": 6.400000000000001e-05, "loss": 0.048635438084602356, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.04984, "step": 17, "tokens/total": 536320, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 7710} +{"epoch": 0.0703125, "grad_norm": 1.6219472885131836, "learning_rate": 6.800000000000001e-05, "loss": 0.03944922983646393, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.04024, "step": 18, "tokens/total": 570744, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 8174} +{"epoch": 0.07421875, "grad_norm": 1.0675303936004639, "learning_rate": 7.2e-05, "loss": 0.040471263229846954, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.0413, "step": 19, "tokens/total": 603984, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 8617} +{"epoch": 0.078125, "grad_norm": 1.018370509147644, "learning_rate": 7.6e-05, "loss": 0.03601411357522011, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.03667, "step": 20, "tokens/total": 637744, "tokens/train_per_sec_per_gpu": 6.52, "tokens/trainable": 9037} +{"epoch": 0.08203125, "grad_norm": 0.6060864329338074, "learning_rate": 8e-05, "loss": 0.02357199229300022, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.02385, "step": 21, "tokens/total": 668240, "tokens/train_per_sec_per_gpu": 10.77, "tokens/trainable": 9503} +{"epoch": 0.0859375, "grad_norm": 0.9379227757453918, "learning_rate": 8.4e-05, "loss": 0.016742059960961342, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.01688, "step": 22, "tokens/total": 700064, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 9972} +{"epoch": 0.08984375, "grad_norm": 1.6366881132125854, "learning_rate": 8.800000000000001e-05, "loss": 0.020147524774074554, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.02035, "step": 23, "tokens/total": 732840, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 10447} +{"epoch": 0.09375, "grad_norm": 1.3981008529663086, "learning_rate": 9.200000000000001e-05, "loss": 0.018401559442281723, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.01857, "step": 24, "tokens/total": 763176, "tokens/train_per_sec_per_gpu": 10.13, "tokens/trainable": 10899} +{"epoch": 0.09765625, "grad_norm": 1.4436801671981812, "learning_rate": 9.6e-05, "loss": 0.024362828582525253, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.02466, "step": 25, "tokens/total": 793464, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 11360} +{"epoch": 0.1015625, "grad_norm": 3.5855233669281006, "learning_rate": 0.0001, "loss": 0.05264014005661011, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.05405, "step": 26, "tokens/total": 820920, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 11775} +{"epoch": 0.10546875, "grad_norm": 1.7416385412216187, "learning_rate": 9.99990636831587e-05, "loss": 0.05233706533908844, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.05373, "step": 27, "tokens/total": 852400, "tokens/train_per_sec_per_gpu": 10.37, "tokens/trainable": 12242} +{"epoch": 0.109375, "grad_norm": 2.571146249771118, "learning_rate": 9.999625477159879e-05, "loss": 0.04750689119100571, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.04865, "step": 28, "tokens/total": 887720, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 12696} +{"epoch": 0.11328125, "grad_norm": 0.7187197208404541, "learning_rate": 9.999157338221051e-05, "loss": 0.02463143691420555, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.02494, "step": 29, "tokens/total": 919640, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 13153} +{"epoch": 0.1171875, "grad_norm": 1.0072499513626099, "learning_rate": 9.998501970980562e-05, "loss": 0.021244579926133156, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.02147, "step": 30, "tokens/total": 949736, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 13624} +{"epoch": 0.12109375, "grad_norm": 0.5468314290046692, "learning_rate": 9.997659402710915e-05, "loss": 0.014794175513088703, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.0149, "step": 31, "tokens/total": 983624, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 14064} +{"epoch": 0.125, "grad_norm": 0.9657878875732422, "learning_rate": 9.996629668474818e-05, "loss": 0.015274718403816223, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.01539, "step": 32, "tokens/total": 1017392, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 14565} +{"epoch": 0.12890625, "grad_norm": 1.6961590051651, "learning_rate": 9.995412811123711e-05, "loss": 0.020734092220664024, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.02095, "step": 33, "tokens/total": 1049600, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 15005} +{"epoch": 0.1328125, "grad_norm": 1.1573313474655151, "learning_rate": 9.994008881295999e-05, "loss": 0.02219044603407383, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.02244, "step": 34, "tokens/total": 1082016, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 15459} +{"epoch": 0.13671875, "grad_norm": 1.206465482711792, "learning_rate": 9.992417937414932e-05, "loss": 0.0225888229906559, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.02285, "step": 35, "tokens/total": 1113368, "tokens/train_per_sec_per_gpu": 10.41, "tokens/trainable": 15960} +{"epoch": 0.140625, "grad_norm": 0.9392801523208618, "learning_rate": 9.99064004568618e-05, "loss": 0.013250188902020454, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.01334, "step": 36, "tokens/total": 1146000, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 16428} +{"epoch": 0.14453125, "grad_norm": 1.0910124778747559, "learning_rate": 9.988675280095074e-05, "loss": 0.016114799305796623, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.01625, "step": 37, "tokens/total": 1176800, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 16884} +{"epoch": 0.1484375, "grad_norm": 1.9194391965866089, "learning_rate": 9.986523722403528e-05, "loss": 0.017150558531284332, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.0173, "step": 38, "tokens/total": 1207232, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 17341} +{"epoch": 0.15234375, "grad_norm": 0.8030540347099304, "learning_rate": 9.984185462146642e-05, "loss": 0.00698848394677043, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00701, "step": 39, "tokens/total": 1239504, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 17786} +{"epoch": 0.15625, "grad_norm": 1.8021860122680664, "learning_rate": 9.98166059662897e-05, "loss": 0.027611931785941124, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.76, "memory/max_allocated (GiB)": 65.76, "ppl": 1.028, "step": 40, "tokens/total": 1272096, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 18262} +{"epoch": 0.16015625, "grad_norm": 2.1310737133026123, "learning_rate": 9.978949230920472e-05, "loss": 0.007026349660009146, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00705, "step": 41, "tokens/total": 1304584, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 18716} +{"epoch": 0.1640625, "grad_norm": 1.5175695419311523, "learning_rate": 9.976051477852141e-05, "loss": 0.023705773055553436, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.02399, "step": 42, "tokens/total": 1337376, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 19157} +{"epoch": 0.16796875, "grad_norm": 3.3011369705200195, "learning_rate": 9.972967458011312e-05, "loss": 0.044193003326654434, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.04518, "step": 43, "tokens/total": 1369576, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 19647} +{"epoch": 0.171875, "grad_norm": 1.4310325384140015, "learning_rate": 9.96969729973664e-05, "loss": 0.045710302889347076, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.04677, "step": 44, "tokens/total": 1407336, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 20119} +{"epoch": 0.17578125, "grad_norm": 0.6940159201622009, "learning_rate": 9.966241139112754e-05, "loss": 0.016563331708312035, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.0167, "step": 45, "tokens/total": 1439392, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 20560} +{"epoch": 0.1796875, "grad_norm": 0.3195332884788513, "learning_rate": 9.96259911996461e-05, "loss": 0.01069785375148058, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01076, "step": 46, "tokens/total": 1471120, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 20992} +{"epoch": 0.18359375, "grad_norm": 1.1725926399230957, "learning_rate": 9.958771393851491e-05, "loss": 0.030896011739969254, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.03138, "step": 47, "tokens/total": 1502272, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 21441} +{"epoch": 0.1875, "grad_norm": 1.1666767597198486, "learning_rate": 9.954758120060702e-05, "loss": 0.028287503868341446, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.02869, "step": 48, "tokens/total": 1534296, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 21901} +{"epoch": 0.19140625, "grad_norm": 0.45164069533348083, "learning_rate": 9.950559465600948e-05, "loss": 0.013161147013306618, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.01325, "step": 49, "tokens/total": 1566544, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 22341} +{"epoch": 0.1953125, "grad_norm": 0.36386141180992126, "learning_rate": 9.946175605195379e-05, "loss": 0.007095211185514927, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00712, "step": 50, "tokens/total": 1601760, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 22833} +{"epoch": 0.19921875, "grad_norm": 0.3774065375328064, "learning_rate": 9.941606721274322e-05, "loss": 0.01018128264695406, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.01023, "step": 51, "tokens/total": 1634312, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 23277} +{"epoch": 0.203125, "grad_norm": 0.6183930039405823, "learning_rate": 9.936853003967685e-05, "loss": 0.014313996769487858, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.01442, "step": 52, "tokens/total": 1668232, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 23759} +{"epoch": 0.20703125, "grad_norm": 1.6092262268066406, "learning_rate": 9.93191465109705e-05, "loss": 0.016688484698534012, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.01683, "step": 53, "tokens/total": 1701160, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 24193} +{"epoch": 0.2109375, "grad_norm": 0.909518301486969, "learning_rate": 9.926791868167438e-05, "loss": 0.01020332332700491, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.01026, "step": 54, "tokens/total": 1735968, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 24619} +{"epoch": 0.21484375, "grad_norm": 1.8369888067245483, "learning_rate": 9.921484868358753e-05, "loss": 0.011584028601646423, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.01165, "step": 55, "tokens/total": 1767624, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 25075} +{"epoch": 0.21875, "grad_norm": 2.4903817176818848, "learning_rate": 9.915993872516924e-05, "loss": 0.02763400599360466, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.02802, "step": 56, "tokens/total": 1800536, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 25519} +{"epoch": 0.22265625, "grad_norm": 0.22497530281543732, "learning_rate": 9.9103191091447e-05, "loss": 0.0005997498519718647, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.0006, "step": 57, "tokens/total": 1830464, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 25966} +{"epoch": 0.2265625, "grad_norm": 0.9195043444633484, "learning_rate": 9.904460814392147e-05, "loss": 0.004160068929195404, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.00417, "step": 58, "tokens/total": 1867568, "tokens/train_per_sec_per_gpu": 7.12, "tokens/trainable": 26423} +{"epoch": 0.23046875, "grad_norm": 1.6133793592453003, "learning_rate": 9.898419232046825e-05, "loss": 0.04068266227841377, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.04152, "step": 59, "tokens/total": 1902424, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 26934} +{"epoch": 0.234375, "grad_norm": 3.121676206588745, "learning_rate": 9.892194613523633e-05, "loss": 0.010118301026523113, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01017, "step": 60, "tokens/total": 1937032, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 27393} +{"epoch": 0.23828125, "grad_norm": 0.7238644361495972, "learning_rate": 9.885787217854357e-05, "loss": 0.003056558081880212, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00306, "step": 61, "tokens/total": 1968960, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 27852} +{"epoch": 0.2421875, "grad_norm": 1.3827385902404785, "learning_rate": 9.879197311676887e-05, "loss": 0.019413378089666367, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.0196, "step": 62, "tokens/total": 2001688, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 28292} +{"epoch": 0.24609375, "grad_norm": 0.9404769539833069, "learning_rate": 9.872425169224113e-05, "loss": 0.022579031065106392, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.02284, "step": 63, "tokens/total": 2034160, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 28798} +{"epoch": 0.25, "grad_norm": 1.2520054578781128, "learning_rate": 9.865471072312528e-05, "loss": 0.020466474816203117, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.02068, "step": 64, "tokens/total": 2069552, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 29283} +{"epoch": 0.25390625, "grad_norm": 0.7099410891532898, "learning_rate": 9.858335310330492e-05, "loss": 0.007657711859792471, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00769, "step": 65, "tokens/total": 2099280, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 29745} +{"epoch": 0.2578125, "grad_norm": 0.7213311791419983, "learning_rate": 9.851018180226185e-05, "loss": 0.016799189150333405, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01694, "step": 66, "tokens/total": 2131280, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 30186} +{"epoch": 0.26171875, "grad_norm": 0.7160526514053345, "learning_rate": 9.843519986495259e-05, "loss": 0.013214158825576305, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.0133, "step": 67, "tokens/total": 2162232, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 30622} +{"epoch": 0.265625, "grad_norm": 0.34394535422325134, "learning_rate": 9.835841041168162e-05, "loss": 0.0075958240777254105, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00762, "step": 68, "tokens/total": 2197080, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 31097} +{"epoch": 0.26953125, "grad_norm": 0.9478030800819397, "learning_rate": 9.82798166379715e-05, "loss": 0.012010117992758751, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.01208, "step": 69, "tokens/total": 2228208, "tokens/train_per_sec_per_gpu": 10.55, "tokens/trainable": 31595} +{"epoch": 0.2734375, "grad_norm": 0.07878953218460083, "learning_rate": 9.819942181443002e-05, "loss": 0.00095759192481637, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.00096, "step": 70, "tokens/total": 2260704, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 32036} +{"epoch": 0.27734375, "grad_norm": 0.8083394169807434, "learning_rate": 9.811722928661392e-05, "loss": 0.004424375481903553, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.00443, "step": 71, "tokens/total": 2288584, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 32428} +{"epoch": 0.28125, "grad_norm": 0.1988888829946518, "learning_rate": 9.803324247488975e-05, "loss": 0.0015440802089869976, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00155, "step": 72, "tokens/total": 2320912, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 32880} +{"epoch": 0.28515625, "grad_norm": 1.2286179065704346, "learning_rate": 9.794746487429161e-05, "loss": 0.018145209178328514, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.01831, "step": 73, "tokens/total": 2352096, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 33323} +{"epoch": 0.2890625, "grad_norm": 0.5450090169906616, "learning_rate": 9.785990005437554e-05, "loss": 0.002222388284280896, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00222, "step": 74, "tokens/total": 2385328, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 33792} +{"epoch": 0.29296875, "grad_norm": 0.9587696194648743, "learning_rate": 9.777055165907117e-05, "loss": 0.02094343490898609, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.02116, "step": 75, "tokens/total": 2416416, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 34223} +{"epoch": 0.296875, "grad_norm": 0.17422634363174438, "learning_rate": 9.767942340652993e-05, "loss": 0.0009302730904892087, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00093, "step": 76, "tokens/total": 2446192, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 34649} +{"epoch": 0.30078125, "grad_norm": 1.9094140529632568, "learning_rate": 9.758651908897035e-05, "loss": 0.041195761412382126, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.04206, "step": 77, "tokens/total": 2478312, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 35094} +{"epoch": 0.3046875, "grad_norm": 2.429379463195801, "learning_rate": 9.749184257252033e-05, "loss": 0.05943134427070618, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.06123, "step": 78, "tokens/total": 2509984, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 35534} +{"epoch": 0.30859375, "grad_norm": 1.2192668914794922, "learning_rate": 9.739539779705614e-05, "loss": 0.024818826466798782, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.02513, "step": 79, "tokens/total": 2546536, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 36029} +{"epoch": 0.3125, "grad_norm": 0.5062951445579529, "learning_rate": 9.729718877603861e-05, "loss": 0.007972100749611855, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.008, "step": 80, "tokens/total": 2578048, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 36469} +{"epoch": 0.31640625, "grad_norm": 1.167993187904358, "learning_rate": 9.719721959634592e-05, "loss": 0.03528548777103424, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.03592, "step": 81, "tokens/total": 2609640, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 36906} +{"epoch": 0.3203125, "grad_norm": 0.8657662272453308, "learning_rate": 9.709549441810375e-05, "loss": 0.007714587729424238, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00774, "step": 82, "tokens/total": 2644408, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 37390} +{"epoch": 0.32421875, "grad_norm": 2.1865603923797607, "learning_rate": 9.699201747451195e-05, "loss": 0.01932365819811821, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01951, "step": 83, "tokens/total": 2675576, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 37838} +{"epoch": 0.328125, "grad_norm": 6.0930867195129395, "learning_rate": 9.688679307166854e-05, "loss": 0.015215419232845306, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.01533, "step": 84, "tokens/total": 2706448, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 38310} +{"epoch": 0.33203125, "grad_norm": 2.2838306427001953, "learning_rate": 9.677982558839042e-05, "loss": 0.03854937106370926, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.0393, "step": 85, "tokens/total": 2738824, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 38789} +{"epoch": 0.3359375, "grad_norm": 0.3014928996562958, "learning_rate": 9.66711194760312e-05, "loss": 0.00953588169068098, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00958, "step": 86, "tokens/total": 2775216, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 39277} +{"epoch": 0.33984375, "grad_norm": 0.5257540345191956, "learning_rate": 9.656067925829593e-05, "loss": 0.012210849672555923, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.01229, "step": 87, "tokens/total": 2804720, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 39737} +{"epoch": 0.34375, "grad_norm": 0.11633944511413574, "learning_rate": 9.644850953105288e-05, "loss": 0.0033296942710876465, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.00334, "step": 88, "tokens/total": 2833944, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 40179} +{"epoch": 0.34765625, "grad_norm": 0.32308685779571533, "learning_rate": 9.633461496214225e-05, "loss": 0.0026228015776723623, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00263, "step": 89, "tokens/total": 2865800, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 40649} +{"epoch": 0.3515625, "grad_norm": 0.3037053644657135, "learning_rate": 9.621900029118195e-05, "loss": 0.005488336086273193, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.0055, "step": 90, "tokens/total": 2894864, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 41080} +{"epoch": 0.35546875, "grad_norm": 0.379432737827301, "learning_rate": 9.610167032937036e-05, "loss": 0.013331079855561256, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.01342, "step": 91, "tokens/total": 2929216, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 41599} +{"epoch": 0.359375, "grad_norm": 0.7861401438713074, "learning_rate": 9.598262995928611e-05, "loss": 0.009017677046358585, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00906, "step": 92, "tokens/total": 2959280, "tokens/train_per_sec_per_gpu": 10.06, "tokens/trainable": 42076} +{"epoch": 0.36328125, "grad_norm": 1.9719332456588745, "learning_rate": 9.586188413468492e-05, "loss": 0.011427424848079681, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01149, "step": 93, "tokens/total": 2993456, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 42522} +{"epoch": 0.3671875, "grad_norm": 0.45879650115966797, "learning_rate": 9.57394378802934e-05, "loss": 0.0047826883383095264, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00479, "step": 94, "tokens/total": 3025752, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 42974} +{"epoch": 0.37109375, "grad_norm": 0.523138165473938, "learning_rate": 9.56152962916e-05, "loss": 0.005802720785140991, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.00582, "step": 95, "tokens/total": 3054192, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 43380} +{"epoch": 0.375, "grad_norm": 0.057836730033159256, "learning_rate": 9.548946453464296e-05, "loss": 0.0006345938891172409, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00063, "step": 96, "tokens/total": 3087160, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 43865} +{"epoch": 0.37890625, "grad_norm": 0.7554764151573181, "learning_rate": 9.53619478457953e-05, "loss": 0.00437867920845747, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00439, "step": 97, "tokens/total": 3121000, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 44328} +{"epoch": 0.3828125, "grad_norm": 0.010660387575626373, "learning_rate": 9.523275153154695e-05, "loss": 0.0001359477755613625, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00014, "step": 98, "tokens/total": 3152272, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 44778} +{"epoch": 0.38671875, "grad_norm": 0.4227249324321747, "learning_rate": 9.51018809682839e-05, "loss": 0.0016907850513234735, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00169, "step": 99, "tokens/total": 3185480, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 45225} +{"epoch": 0.390625, "grad_norm": 0.6491655111312866, "learning_rate": 9.49693416020645e-05, "loss": 0.010564007796347141, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.01062, "step": 100, "tokens/total": 3219360, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 45678} +{"epoch": 0.39453125, "grad_norm": 0.28581100702285767, "learning_rate": 9.483513894839276e-05, "loss": 0.0016267925966531038, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00163, "step": 101, "tokens/total": 3251368, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 46125} +{"epoch": 0.3984375, "grad_norm": 1.028909683227539, "learning_rate": 9.469927859198888e-05, "loss": 0.01776937022805214, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01793, "step": 102, "tokens/total": 3284184, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 46612} +{"epoch": 0.40234375, "grad_norm": 0.031058812513947487, "learning_rate": 9.456176618655689e-05, "loss": 0.00021491775987669826, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00021, "step": 103, "tokens/total": 3314752, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 47059} +{"epoch": 0.40625, "grad_norm": 0.6987336874008179, "learning_rate": 9.442260745454927e-05, "loss": 0.004446749109774828, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00446, "step": 104, "tokens/total": 3349736, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 47536} +{"epoch": 0.41015625, "grad_norm": 1.0073214769363403, "learning_rate": 9.428180818692884e-05, "loss": 0.00726565346121788, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00729, "step": 105, "tokens/total": 3384224, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 48037} +{"epoch": 0.4140625, "grad_norm": 0.12182477116584778, "learning_rate": 9.413937424292791e-05, "loss": 0.0004033746663480997, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.0004, "step": 106, "tokens/total": 3420808, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 48491} +{"epoch": 0.41796875, "grad_norm": 0.6656859517097473, "learning_rate": 9.399531154980424e-05, "loss": 0.005425583571195602, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00544, "step": 107, "tokens/total": 3453512, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 48940} +{"epoch": 0.421875, "grad_norm": 0.18209248781204224, "learning_rate": 9.384962610259455e-05, "loss": 0.0012126248329877853, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00121, "step": 108, "tokens/total": 3486192, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 49389} +{"epoch": 0.42578125, "grad_norm": 0.34885913133621216, "learning_rate": 9.370232396386494e-05, "loss": 0.003417135216295719, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00342, "step": 109, "tokens/total": 3515384, "tokens/train_per_sec_per_gpu": 10.44, "tokens/trainable": 49838} +{"epoch": 0.4296875, "grad_norm": 0.7507489919662476, "learning_rate": 9.355341126345868e-05, "loss": 0.004546954296529293, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00456, "step": 110, "tokens/total": 3548264, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 50310} +{"epoch": 0.43359375, "grad_norm": 0.22768281400203705, "learning_rate": 9.340289419824107e-05, "loss": 0.0027937067206948996, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.0028, "step": 111, "tokens/total": 3579288, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 50755} +{"epoch": 0.4375, "grad_norm": 0.6052107810974121, "learning_rate": 9.325077903184159e-05, "loss": 0.010212267749011517, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.01026, "step": 112, "tokens/total": 3611240, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 51213} +{"epoch": 0.44140625, "grad_norm": 0.05136172100901604, "learning_rate": 9.30970720943932e-05, "loss": 0.00014994248340371996, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00015, "step": 113, "tokens/total": 3644136, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 51660} +{"epoch": 0.4453125, "grad_norm": 2.2271125316619873, "learning_rate": 9.2941779782269e-05, "loss": 0.008876129053533077, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00892, "step": 114, "tokens/total": 3676368, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 52133} +{"epoch": 0.44921875, "grad_norm": 1.3910104036331177, "learning_rate": 9.278490855781596e-05, "loss": 0.008210944943130016, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00824, "step": 115, "tokens/total": 3707896, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 52580} +{"epoch": 0.453125, "grad_norm": 0.4979943037033081, "learning_rate": 9.262646494908604e-05, "loss": 0.003800376085564494, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00381, "step": 116, "tokens/total": 3742296, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 53033} +{"epoch": 0.45703125, "grad_norm": 1.0015233755111694, "learning_rate": 9.246645554956457e-05, "loss": 0.020017284899950027, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.02022, "step": 117, "tokens/total": 3776456, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 53517} +{"epoch": 0.4609375, "grad_norm": 0.9948551654815674, "learning_rate": 9.230488701789578e-05, "loss": 0.004522592760622501, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00453, "step": 118, "tokens/total": 3811688, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 53967} +{"epoch": 0.46484375, "grad_norm": 0.14162909984588623, "learning_rate": 9.214176607760577e-05, "loss": 0.0020346923265606165, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00204, "step": 119, "tokens/total": 3847200, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 54430} +{"epoch": 0.46875, "grad_norm": 0.39979708194732666, "learning_rate": 9.197709951682268e-05, "loss": 0.010470978915691376, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.01053, "step": 120, "tokens/total": 3877800, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 54896} +{"epoch": 0.47265625, "grad_norm": 1.2159241437911987, "learning_rate": 9.181089418799428e-05, "loss": 0.016084477305412292, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.01621, "step": 121, "tokens/total": 3910976, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 55379} +{"epoch": 0.4765625, "grad_norm": 0.4352659285068512, "learning_rate": 9.164315700760271e-05, "loss": 0.006844333838671446, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00687, "step": 122, "tokens/total": 3943768, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 55803} +{"epoch": 0.48046875, "grad_norm": 5.911741256713867, "learning_rate": 9.147389495587671e-05, "loss": 0.013668426312506199, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01376, "step": 123, "tokens/total": 3973816, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 56249} +{"epoch": 0.484375, "grad_norm": 1.1447240114212036, "learning_rate": 9.130311507650116e-05, "loss": 0.01663239486515522, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.01677, "step": 124, "tokens/total": 4004704, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 56713} +{"epoch": 0.48828125, "grad_norm": 1.0569020509719849, "learning_rate": 9.113082447632394e-05, "loss": 0.021982934325933456, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.02223, "step": 125, "tokens/total": 4036040, "tokens/train_per_sec_per_gpu": 10.59, "tokens/trainable": 57196} +{"epoch": 0.4921875, "grad_norm": 0.8642127513885498, "learning_rate": 9.09570303250602e-05, "loss": 0.00732447998598218, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00735, "step": 126, "tokens/total": 4066792, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 57680} +{"epoch": 0.49609375, "grad_norm": 3.271244525909424, "learning_rate": 9.078173985499394e-05, "loss": 0.0037689961027354, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00378, "step": 127, "tokens/total": 4096112, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 58110} +{"epoch": 0.5, "grad_norm": 0.6168667078018188, "learning_rate": 9.060496036067713e-05, "loss": 0.008628081530332565, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00867, "step": 128, "tokens/total": 4131472, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 58534} +{"epoch": 0.50390625, "grad_norm": 0.18315109610557556, "learning_rate": 9.042669919862615e-05, "loss": 0.0032129588071256876, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00322, "step": 129, "tokens/total": 4162912, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 58998} +{"epoch": 0.5078125, "grad_norm": 0.4149618148803711, "learning_rate": 9.024696378701557e-05, "loss": 0.0040068733505904675, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00401, "step": 130, "tokens/total": 4195768, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 59448} +{"epoch": 0.51171875, "grad_norm": 0.15302760899066925, "learning_rate": 9.006576160536948e-05, "loss": 0.0014866620767861605, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00149, "step": 131, "tokens/total": 4227192, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 59906} +{"epoch": 0.515625, "grad_norm": 2.7171857357025146, "learning_rate": 8.988310019425035e-05, "loss": 0.008593314327299595, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00863, "step": 132, "tokens/total": 4263304, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 60350} +{"epoch": 0.51953125, "grad_norm": 1.2315974235534668, "learning_rate": 8.969898715494506e-05, "loss": 0.011854372918605804, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01192, "step": 133, "tokens/total": 4297352, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 60831} +{"epoch": 0.5234375, "grad_norm": 0.31457674503326416, "learning_rate": 8.951343014914869e-05, "loss": 0.007473263889551163, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.0075, "step": 134, "tokens/total": 4332152, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 61305} +{"epoch": 0.52734375, "grad_norm": 6.253246784210205, "learning_rate": 8.932643689864568e-05, "loss": 0.010792153887450695, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.01085, "step": 135, "tokens/total": 4365272, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 61792} +{"epoch": 0.53125, "grad_norm": 0.10361011326313019, "learning_rate": 8.913801518498845e-05, "loss": 0.0008364556124433875, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00084, "step": 136, "tokens/total": 4399928, "tokens/train_per_sec_per_gpu": 6.98, "tokens/trainable": 62253} +{"epoch": 0.53515625, "grad_norm": 0.4883211851119995, "learning_rate": 8.894817284917364e-05, "loss": 0.00851798802614212, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00855, "step": 137, "tokens/total": 4434336, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 62707} +{"epoch": 0.5390625, "grad_norm": 1.0248528718948364, "learning_rate": 8.875691779131569e-05, "loss": 0.018130647018551826, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.0183, "step": 138, "tokens/total": 4468872, "tokens/train_per_sec_per_gpu": 6.47, "tokens/trainable": 63145} +{"epoch": 0.54296875, "grad_norm": 0.6904343962669373, "learning_rate": 8.856425797031829e-05, "loss": 0.013388886116445065, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01348, "step": 139, "tokens/total": 4502416, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 63587} +{"epoch": 0.546875, "grad_norm": 0.1456335186958313, "learning_rate": 8.837020140354295e-05, "loss": 0.002086236607283354, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00209, "step": 140, "tokens/total": 4536024, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 64052} +{"epoch": 0.55078125, "grad_norm": 0.4557308554649353, "learning_rate": 8.817475616647554e-05, "loss": 0.005815692245960236, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00583, "step": 141, "tokens/total": 4568960, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 64526} +{"epoch": 0.5546875, "grad_norm": 0.5417463183403015, "learning_rate": 8.797793039239017e-05, "loss": 0.014805496670305729, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.01492, "step": 142, "tokens/total": 4601656, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 64983} +{"epoch": 0.55859375, "grad_norm": 0.878449022769928, "learning_rate": 8.777973227201069e-05, "loss": 0.010475190356373787, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.01053, "step": 143, "tokens/total": 4636136, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 65492} +{"epoch": 0.5625, "grad_norm": 0.2563984990119934, "learning_rate": 8.758017005316988e-05, "loss": 0.006462044082581997, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00648, "step": 144, "tokens/total": 4670160, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 65925} +{"epoch": 0.56640625, "grad_norm": 0.38753849267959595, "learning_rate": 8.737925204046629e-05, "loss": 0.011333402246236801, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.0114, "step": 145, "tokens/total": 4703792, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 66383} +{"epoch": 0.5703125, "grad_norm": 0.780264675617218, "learning_rate": 8.717698659491851e-05, "loss": 0.00510529987514019, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00512, "step": 146, "tokens/total": 4737640, "tokens/train_per_sec_per_gpu": 7.11, "tokens/trainable": 66840} +{"epoch": 0.57421875, "grad_norm": 0.5245252251625061, "learning_rate": 8.697338213361735e-05, "loss": 0.012531804852187634, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01261, "step": 147, "tokens/total": 4770704, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 67297} +{"epoch": 0.578125, "grad_norm": 0.24315930902957916, "learning_rate": 8.676844712937552e-05, "loss": 0.006579314824193716, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.0066, "step": 148, "tokens/total": 4807376, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 67769} +{"epoch": 0.58203125, "grad_norm": 0.7559230923652649, "learning_rate": 8.656219011037509e-05, "loss": 0.016810227185487747, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01695, "step": 149, "tokens/total": 4840728, "tokens/train_per_sec_per_gpu": 6.69, "tokens/trainable": 68257} +{"epoch": 0.5859375, "grad_norm": 0.23798292875289917, "learning_rate": 8.63546196598125e-05, "loss": 0.002012595534324646, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00201, "step": 150, "tokens/total": 4873880, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 68706} +{"epoch": 0.58984375, "grad_norm": 0.8809586763381958, "learning_rate": 8.614574441554145e-05, "loss": 0.0022340782452374697, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00224, "step": 151, "tokens/total": 4904512, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 69131} +{"epoch": 0.59375, "grad_norm": 1.0374332666397095, "learning_rate": 8.593557306971349e-05, "loss": 0.0032807309180498123, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00329, "step": 152, "tokens/total": 4937048, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 69586} +{"epoch": 0.59765625, "grad_norm": 0.17572350800037384, "learning_rate": 8.572411436841618e-05, "loss": 0.0011859629303216934, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00119, "step": 153, "tokens/total": 4968224, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 70061} +{"epoch": 0.6015625, "grad_norm": 0.2845759689807892, "learning_rate": 8.551137711130922e-05, "loss": 0.0018781210528686643, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00188, "step": 154, "tokens/total": 4997272, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 70467} +{"epoch": 0.60546875, "grad_norm": 0.0475124716758728, "learning_rate": 8.529737015125824e-05, "loss": 0.00040228376747108996, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.0004, "step": 155, "tokens/total": 5031072, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 70933} +{"epoch": 0.609375, "grad_norm": 1.9162992238998413, "learning_rate": 8.508210239396639e-05, "loss": 0.029181238263845444, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.02961, "step": 156, "tokens/total": 5063808, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 71382} +{"epoch": 0.61328125, "grad_norm": 0.472818523645401, "learning_rate": 8.486558279760375e-05, "loss": 0.0027511066291481256, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00275, "step": 157, "tokens/total": 5094032, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 71808} +{"epoch": 0.6171875, "grad_norm": 1.6822749376296997, "learning_rate": 8.464782037243449e-05, "loss": 0.007599940523505211, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00763, "step": 158, "tokens/total": 5125664, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 72249} +{"epoch": 0.62109375, "grad_norm": 1.4170866012573242, "learning_rate": 8.442882418044202e-05, "loss": 0.00231572357006371, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00232, "step": 159, "tokens/total": 5156952, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 72726} +{"epoch": 0.625, "grad_norm": 1.0304460525512695, "learning_rate": 8.420860333495179e-05, "loss": 0.004054890479892492, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00406, "step": 160, "tokens/total": 5187120, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 73148} +{"epoch": 0.62890625, "grad_norm": 0.0861649215221405, "learning_rate": 8.398716700025208e-05, "loss": 0.0004486228572204709, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00045, "step": 161, "tokens/total": 5219080, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 73600} +{"epoch": 0.6328125, "grad_norm": 0.8162397146224976, "learning_rate": 8.376452439121266e-05, "loss": 0.005480658262968063, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.0055, "step": 162, "tokens/total": 5248680, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 74068} +{"epoch": 0.63671875, "grad_norm": 0.025036020204424858, "learning_rate": 8.354068477290124e-05, "loss": 0.00017588163609616458, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00018, "step": 163, "tokens/total": 5280104, "tokens/train_per_sec_per_gpu": 10.27, "tokens/trainable": 74527} +{"epoch": 0.640625, "grad_norm": 0.2183261215686798, "learning_rate": 8.331565746019807e-05, "loss": 0.0012128445087000728, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00121, "step": 164, "tokens/total": 5311328, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 74992} +{"epoch": 0.64453125, "grad_norm": 3.9487059116363525, "learning_rate": 8.308945181740812e-05, "loss": 0.01977401226758957, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.01997, "step": 165, "tokens/total": 5343960, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 75442} +{"epoch": 0.6484375, "grad_norm": 0.4532577097415924, "learning_rate": 8.286207725787153e-05, "loss": 0.010736935772001743, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.01079, "step": 166, "tokens/total": 5377312, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 75887} +{"epoch": 0.65234375, "grad_norm": 8.748824119567871, "learning_rate": 8.263354324357182e-05, "loss": 0.013092179782688618, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.01318, "step": 167, "tokens/total": 5409824, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 76331} +{"epoch": 0.65625, "grad_norm": 0.27062568068504333, "learning_rate": 8.240385928474219e-05, "loss": 0.002099117496982217, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.0021, "step": 168, "tokens/total": 5440904, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 76745} +{"epoch": 0.66015625, "grad_norm": 1.7256834506988525, "learning_rate": 8.217303493946967e-05, "loss": 0.020528443157672882, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.02074, "step": 169, "tokens/total": 5474920, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 77201} +{"epoch": 0.6640625, "grad_norm": 0.5393511056900024, "learning_rate": 8.194107981329746e-05, "loss": 0.004844017792493105, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00486, "step": 170, "tokens/total": 5511856, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 77655} +{"epoch": 0.66796875, "grad_norm": 0.9536205530166626, "learning_rate": 8.170800355882518e-05, "loss": 0.006854225881397724, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00688, "step": 171, "tokens/total": 5544104, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 78122} +{"epoch": 0.671875, "grad_norm": 2.2425761222839355, "learning_rate": 8.147381587530713e-05, "loss": 0.018021494150161743, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01818, "step": 172, "tokens/total": 5577344, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 78576} +{"epoch": 0.67578125, "grad_norm": 0.9310670495033264, "learning_rate": 8.123852650824877e-05, "loss": 0.008047185838222504, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00808, "step": 173, "tokens/total": 5610024, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 79059} +{"epoch": 0.6796875, "grad_norm": 1.5290974378585815, "learning_rate": 8.100214524900103e-05, "loss": 0.008589663542807102, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00863, "step": 174, "tokens/total": 5643184, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 79498} +{"epoch": 0.68359375, "grad_norm": 0.7947601079940796, "learning_rate": 8.076468193435301e-05, "loss": 0.013502977788448334, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.01359, "step": 175, "tokens/total": 5672824, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 79922} +{"epoch": 0.6875, "grad_norm": 0.23027248680591583, "learning_rate": 8.052614644612253e-05, "loss": 0.0039336541667580605, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00394, "step": 176, "tokens/total": 5705576, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 80383} +{"epoch": 0.69140625, "grad_norm": 0.04238725081086159, "learning_rate": 8.028654871074489e-05, "loss": 0.0006386310560628772, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00064, "step": 177, "tokens/total": 5737840, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 80824} +{"epoch": 0.6953125, "grad_norm": 0.15484179556369781, "learning_rate": 8.004589869885986e-05, "loss": 0.0034990892745554447, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00351, "step": 178, "tokens/total": 5767552, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 81243} +{"epoch": 0.69921875, "grad_norm": 0.3233547508716583, "learning_rate": 7.980420642489674e-05, "loss": 0.00940670259296894, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00945, "step": 179, "tokens/total": 5802512, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 81716} +{"epoch": 0.703125, "grad_norm": 0.738247811794281, "learning_rate": 7.95614819466576e-05, "loss": 0.007044158410280943, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00707, "step": 180, "tokens/total": 5834984, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 82181} +{"epoch": 0.70703125, "grad_norm": 1.2879854440689087, "learning_rate": 7.931773536489872e-05, "loss": 0.0019345785258337855, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00194, "step": 181, "tokens/total": 5867336, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 82626} +{"epoch": 0.7109375, "grad_norm": 0.39126908779144287, "learning_rate": 7.907297682291035e-05, "loss": 0.006331650074571371, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00635, "step": 182, "tokens/total": 5899232, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 83089} +{"epoch": 0.71484375, "grad_norm": 0.3492918014526367, "learning_rate": 7.882721650609442e-05, "loss": 0.00419978704303503, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00421, "step": 183, "tokens/total": 5934816, "tokens/train_per_sec_per_gpu": 6.78, "tokens/trainable": 83590} +{"epoch": 0.71875, "grad_norm": 0.20798547565937042, "learning_rate": 7.85804646415409e-05, "loss": 0.003952442202717066, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.00396, "step": 184, "tokens/total": 5968120, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 84046} +{"epoch": 0.72265625, "grad_norm": 0.22962047159671783, "learning_rate": 7.833273149760207e-05, "loss": 0.004357465542852879, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00437, "step": 185, "tokens/total": 5998600, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 84517} +{"epoch": 0.7265625, "grad_norm": 0.7952356934547424, "learning_rate": 7.808402738346527e-05, "loss": 0.018292343243956566, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01846, "step": 186, "tokens/total": 6033832, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 84974} +{"epoch": 0.73046875, "grad_norm": 0.24339374899864197, "learning_rate": 7.783436264872382e-05, "loss": 0.001428151037544012, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00143, "step": 187, "tokens/total": 6068192, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 85460} +{"epoch": 0.734375, "grad_norm": 0.30019611120224, "learning_rate": 7.758374768294647e-05, "loss": 0.00702957808971405, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00705, "step": 188, "tokens/total": 6101464, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 85939} +{"epoch": 0.73828125, "grad_norm": 0.44086936116218567, "learning_rate": 7.733219291524489e-05, "loss": 0.010100271552801132, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.01015, "step": 189, "tokens/total": 6133008, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 86377} +{"epoch": 0.7421875, "grad_norm": 0.0626629963517189, "learning_rate": 7.707970881383977e-05, "loss": 0.001449858071282506, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.00145, "step": 190, "tokens/total": 6163048, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 86834} +{"epoch": 0.74609375, "grad_norm": 0.2441553771495819, "learning_rate": 7.682630588562518e-05, "loss": 0.003820787649601698, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00383, "step": 191, "tokens/total": 6196072, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 87265} +{"epoch": 0.75, "grad_norm": 0.3010576367378235, "learning_rate": 7.657199467573129e-05, "loss": 0.003515787422657013, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00352, "step": 192, "tokens/total": 6227312, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 87747} +{"epoch": 0.75390625, "grad_norm": 0.06623072177171707, "learning_rate": 7.631678576708561e-05, "loss": 0.0012438015546649694, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00124, "step": 193, "tokens/total": 6260240, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 88239} +{"epoch": 0.7578125, "grad_norm": 0.020404815673828125, "learning_rate": 7.606068977997255e-05, "loss": 0.00037127139512449503, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00037, "step": 194, "tokens/total": 6294280, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 88718} +{"epoch": 0.76171875, "grad_norm": 0.07147902250289917, "learning_rate": 7.580371737159148e-05, "loss": 0.000746965582948178, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00075, "step": 195, "tokens/total": 6323368, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 89129} +{"epoch": 0.765625, "grad_norm": 0.02040976472198963, "learning_rate": 7.554587923561324e-05, "loss": 0.0001872300636023283, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00019, "step": 196, "tokens/total": 6354960, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 89567} +{"epoch": 0.76953125, "grad_norm": 0.2253779172897339, "learning_rate": 7.528718610173511e-05, "loss": 0.0008828976424410939, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.00088, "step": 197, "tokens/total": 6387208, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 89988} +{"epoch": 0.7734375, "grad_norm": 0.18206360936164856, "learning_rate": 7.502764873523431e-05, "loss": 0.0010578801156952977, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00106, "step": 198, "tokens/total": 6418720, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 90434} +{"epoch": 0.77734375, "grad_norm": 0.19160586595535278, "learning_rate": 7.476727793652011e-05, "loss": 0.0010771253146231174, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00108, "step": 199, "tokens/total": 6449184, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 90913} +{"epoch": 0.78125, "grad_norm": 0.16960078477859497, "learning_rate": 7.450608454068415e-05, "loss": 0.001300798263400793, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.0013, "step": 200, "tokens/total": 6481824, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 91355} +{"epoch": 0.78515625, "grad_norm": 0.577411949634552, "learning_rate": 7.424407941704987e-05, "loss": 0.003797393525019288, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.0038, "step": 201, "tokens/total": 6515896, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 91856} +{"epoch": 0.7890625, "grad_norm": 2.422013521194458, "learning_rate": 7.398127346871986e-05, "loss": 0.006983944680541754, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00701, "step": 202, "tokens/total": 6548784, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 92311} +{"epoch": 0.79296875, "grad_norm": 0.008678768761456013, "learning_rate": 7.371767763212238e-05, "loss": 6.908440991537645e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00007, "step": 203, "tokens/total": 6580256, "tokens/train_per_sec_per_gpu": 10.16, "tokens/trainable": 92764} +{"epoch": 0.796875, "grad_norm": 0.09056023508310318, "learning_rate": 7.345330287655617e-05, "loss": 0.0007011370034888387, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.0007, "step": 204, "tokens/total": 6613976, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 93227} +{"epoch": 0.80078125, "grad_norm": 0.03130093589425087, "learning_rate": 7.31881602037339e-05, "loss": 5.395166954258457e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.00005, "step": 205, "tokens/total": 6644552, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 93675} +{"epoch": 0.8046875, "grad_norm": 0.9986370801925659, "learning_rate": 7.29222606473245e-05, "loss": 0.010276943445205688, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.01033, "step": 206, "tokens/total": 6675056, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 94120} +{"epoch": 0.80859375, "grad_norm": 0.003077535890042782, "learning_rate": 7.265561527249383e-05, "loss": 4.1742881876416504e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00004, "step": 207, "tokens/total": 6706624, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 94557} +{"epoch": 0.8125, "grad_norm": 0.010239548049867153, "learning_rate": 7.238823517544436e-05, "loss": 6.878904241602868e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00007, "step": 208, "tokens/total": 6741008, "tokens/train_per_sec_per_gpu": 10.01, "tokens/trainable": 95035} +{"epoch": 0.81640625, "grad_norm": 0.3272138833999634, "learning_rate": 7.212013148295333e-05, "loss": 0.0025587650015950203, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00256, "step": 209, "tokens/total": 6774872, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 95517} +{"epoch": 0.8203125, "grad_norm": 0.8727256655693054, "learning_rate": 7.185131535190975e-05, "loss": 0.024163801223039627, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.02446, "step": 210, "tokens/total": 6805816, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 96026} +{"epoch": 0.82421875, "grad_norm": 0.10065029561519623, "learning_rate": 7.158179796885005e-05, "loss": 9.511820826446638e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.0001, "step": 211, "tokens/total": 6837592, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 96477} +{"epoch": 0.828125, "grad_norm": 0.5070984363555908, "learning_rate": 7.131159054949273e-05, "loss": 0.005963616073131561, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00598, "step": 212, "tokens/total": 6874384, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 96951} +{"epoch": 0.83203125, "grad_norm": 0.9889188408851624, "learning_rate": 7.104070433827139e-05, "loss": 0.0022374303080141544, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00224, "step": 213, "tokens/total": 6905872, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 97350} +{"epoch": 0.8359375, "grad_norm": 0.5320567488670349, "learning_rate": 7.076915060786705e-05, "loss": 0.007136400323361158, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.00716, "step": 214, "tokens/total": 6937984, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 97792} +{"epoch": 0.83984375, "grad_norm": 0.6934662461280823, "learning_rate": 7.049694065873882e-05, "loss": 0.007576661184430122, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00761, "step": 215, "tokens/total": 6970704, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 98257} +{"epoch": 0.84375, "grad_norm": 0.609321653842926, "learning_rate": 7.022408581865382e-05, "loss": 0.003974556922912598, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00398, "step": 216, "tokens/total": 7004616, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 98731} +{"epoch": 0.84765625, "grad_norm": 0.06983970105648041, "learning_rate": 6.99505974422157e-05, "loss": 0.00035802353522740304, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00036, "step": 217, "tokens/total": 7036880, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 99212} +{"epoch": 0.8515625, "grad_norm": 0.3957507312297821, "learning_rate": 6.967648691039213e-05, "loss": 0.0032922718673944473, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.0033, "step": 218, "tokens/total": 7068208, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 99654} +{"epoch": 0.85546875, "grad_norm": 8.664958000183105, "learning_rate": 6.940176563004123e-05, "loss": 0.002364075742661953, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00237, "step": 219, "tokens/total": 7098112, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 100086} +{"epoch": 0.859375, "grad_norm": 0.8820739984512329, "learning_rate": 6.912644503343682e-05, "loss": 0.017134224995970726, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01728, "step": 220, "tokens/total": 7132432, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 100524} +{"epoch": 0.86328125, "grad_norm": 4.043310642242432, "learning_rate": 6.885053657779273e-05, "loss": 0.015286924317479134, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.0154, "step": 221, "tokens/total": 7164016, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 100996} +{"epoch": 0.8671875, "grad_norm": 1.5245273113250732, "learning_rate": 6.857405174478604e-05, "loss": 0.017334669828414917, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.01749, "step": 222, "tokens/total": 7198344, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 101449} +{"epoch": 0.87109375, "grad_norm": 0.3158121407032013, "learning_rate": 6.82970020400792e-05, "loss": 0.005412735976278782, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00543, "step": 223, "tokens/total": 7232192, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 101905} +{"epoch": 0.875, "grad_norm": 0.4962981939315796, "learning_rate": 6.801939899284132e-05, "loss": 0.002139876363798976, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00214, "step": 224, "tokens/total": 7266984, "tokens/train_per_sec_per_gpu": 9.88, "tokens/trainable": 102411} +{"epoch": 0.87890625, "grad_norm": 0.39842668175697327, "learning_rate": 6.774125415526827e-05, "loss": 0.0031997710466384888, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.0032, "step": 225, "tokens/total": 7299512, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 102882} +{"epoch": 0.8828125, "grad_norm": 0.07509301602840424, "learning_rate": 6.746257910210214e-05, "loss": 0.000719614268746227, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00072, "step": 226, "tokens/total": 7331120, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 103332} +{"epoch": 0.88671875, "grad_norm": 0.18970537185668945, "learning_rate": 6.718338543014937e-05, "loss": 0.0016078815096989274, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00161, "step": 227, "tokens/total": 7360504, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 103790} +{"epoch": 0.890625, "grad_norm": 0.0274028442800045, "learning_rate": 6.69036847577983e-05, "loss": 0.0001558746735099703, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00016, "step": 228, "tokens/total": 7390760, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 104246} +{"epoch": 0.89453125, "grad_norm": 0.23100557923316956, "learning_rate": 6.662348872453553e-05, "loss": 0.002127659972757101, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00213, "step": 229, "tokens/total": 7423352, "tokens/train_per_sec_per_gpu": 10.57, "tokens/trainable": 104707} +{"epoch": 0.8984375, "grad_norm": 1.0105732679367065, "learning_rate": 6.63428089904618e-05, "loss": 0.005608569830656052, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00562, "step": 230, "tokens/total": 7457152, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 105167} +{"epoch": 0.90234375, "grad_norm": 0.39417386054992676, "learning_rate": 6.60616572358065e-05, "loss": 0.004491505678743124, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.0045, "step": 231, "tokens/total": 7488648, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 105576} +{"epoch": 0.90625, "grad_norm": 0.5517076849937439, "learning_rate": 6.578004516044172e-05, "loss": 0.006592917256057262, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00661, "step": 232, "tokens/total": 7522160, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 106043} +{"epoch": 0.91015625, "grad_norm": 0.2433277815580368, "learning_rate": 6.549798448339548e-05, "loss": 0.0019948913250118494, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.002, "step": 233, "tokens/total": 7557336, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 106506} +{"epoch": 0.9140625, "grad_norm": 0.07451238483190536, "learning_rate": 6.521548694236384e-05, "loss": 0.0006025153561495245, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.0006, "step": 234, "tokens/total": 7590432, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 106951} +{"epoch": 0.91796875, "grad_norm": 1.3321301937103271, "learning_rate": 6.493256429322259e-05, "loss": 0.01756269671022892, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.01772, "step": 235, "tokens/total": 7623240, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 107382} +{"epoch": 0.921875, "grad_norm": 0.2823522686958313, "learning_rate": 6.464922830953799e-05, "loss": 0.002869800664484501, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00287, "step": 236, "tokens/total": 7657968, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 107814} +{"epoch": 0.92578125, "grad_norm": 0.68572598695755, "learning_rate": 6.436549078207688e-05, "loss": 0.007511276751756668, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00754, "step": 237, "tokens/total": 7689320, "tokens/train_per_sec_per_gpu": 10.22, "tokens/trainable": 108274} +{"epoch": 0.9296875, "grad_norm": 0.6752313375473022, "learning_rate": 6.408136351831592e-05, "loss": 0.004511378705501556, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00452, "step": 238, "tokens/total": 7721568, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 108714} +{"epoch": 0.93359375, "grad_norm": 0.08202631771564484, "learning_rate": 6.379685834195036e-05, "loss": 0.0006028208881616592, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.0006, "step": 239, "tokens/total": 7756464, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 109220} +{"epoch": 0.9375, "grad_norm": 0.06374165415763855, "learning_rate": 6.351198709240186e-05, "loss": 0.0005584401078522205, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00056, "step": 240, "tokens/total": 7791616, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 109672} +{"epoch": 0.94140625, "grad_norm": 1.2206424474716187, "learning_rate": 6.32267616243259e-05, "loss": 0.0013264388544484973, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00133, "step": 241, "tokens/total": 7823520, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 110135} +{"epoch": 0.9453125, "grad_norm": 0.0015146207297220826, "learning_rate": 6.294119380711849e-05, "loss": 3.1873096304479986e-05, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00003, "step": 242, "tokens/total": 7854424, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 110563} +{"epoch": 0.94921875, "grad_norm": 0.09411055594682693, "learning_rate": 6.265529552442209e-05, "loss": 0.0004007034585811198, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.0004, "step": 243, "tokens/total": 7887160, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 111049} +{"epoch": 0.953125, "grad_norm": 0.7551202178001404, "learning_rate": 6.236907867363127e-05, "loss": 0.0030224043875932693, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00303, "step": 244, "tokens/total": 7918736, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 111495} +{"epoch": 0.95703125, "grad_norm": 0.15987719595432281, "learning_rate": 6.208255516539749e-05, "loss": 0.0006659028586000204, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00067, "step": 245, "tokens/total": 7949448, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 111968} +{"epoch": 0.9609375, "grad_norm": 0.2998705506324768, "learning_rate": 6.179573692313344e-05, "loss": 0.0009331938344985247, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00093, "step": 246, "tokens/total": 7984680, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 112416} +{"epoch": 0.96484375, "grad_norm": 0.2837025821208954, "learning_rate": 6.150863588251694e-05, "loss": 0.0026929648593068123, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.0027, "step": 247, "tokens/total": 8019816, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 112882} +{"epoch": 0.96875, "grad_norm": 0.5971419215202332, "learning_rate": 6.122126399099419e-05, "loss": 0.018822243437170982, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.019, "step": 248, "tokens/total": 8056272, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 113390} +{"epoch": 0.97265625, "grad_norm": 0.13754761219024658, "learning_rate": 6.0933633207282615e-05, "loss": 0.0004868851974606514, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00049, "step": 249, "tokens/total": 8090224, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 113904} +{"epoch": 0.9765625, "grad_norm": 0.5140104293823242, "learning_rate": 6.064575550087316e-05, "loss": 0.0048652649857103825, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00488, "step": 250, "tokens/total": 8124528, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 114351} +{"epoch": 0.98046875, "grad_norm": 0.023654861375689507, "learning_rate": 6.0357642851532245e-05, "loss": 3.768194073927589e-05, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00004, "step": 251, "tokens/total": 8159688, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 114842} +{"epoch": 0.984375, "grad_norm": 0.18568114936351776, "learning_rate": 6.0069307248803294e-05, "loss": 0.0010823197662830353, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00108, "step": 252, "tokens/total": 8191800, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 115263} +{"epoch": 0.98828125, "grad_norm": 0.16117927432060242, "learning_rate": 5.9780760691507635e-05, "loss": 0.001209760783240199, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00121, "step": 253, "tokens/total": 8221752, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 115703} +{"epoch": 0.9921875, "grad_norm": 0.5030807256698608, "learning_rate": 5.9492015187245334e-05, "loss": 0.004646642133593559, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00466, "step": 254, "tokens/total": 8255648, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 116157} +{"epoch": 0.99609375, "grad_norm": 0.707248330116272, "learning_rate": 5.920308275189541e-05, "loss": 0.01014221552759409, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.01019, "step": 255, "tokens/total": 8286664, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 116567} +{"epoch": 1.0, "grad_norm": 0.006301385350525379, "learning_rate": 5.8913975409115874e-05, "loss": 0.00010897110041696578, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00011, "step": 256, "tokens/total": 8318368, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 117030} +{"epoch": 1.00390625, "grad_norm": 0.14343687891960144, "learning_rate": 5.8624705189843395e-05, "loss": 0.0010297894477844238, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00103, "step": 257, "tokens/total": 8353336, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 117517} +{"epoch": 1.0078125, "grad_norm": 0.13553519546985626, "learning_rate": 5.833528413179249e-05, "loss": 0.00030589583911933005, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00031, "step": 258, "tokens/total": 8389112, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 118015} +{"epoch": 1.01171875, "grad_norm": 0.11555363982915878, "learning_rate": 5.80457242789548e-05, "loss": 0.0011719211470335722, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00117, "step": 259, "tokens/total": 8426000, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 118520} +{"epoch": 1.015625, "grad_norm": 0.7873225808143616, "learning_rate": 5.77560376810977e-05, "loss": 0.010044656693935394, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.0101, "step": 260, "tokens/total": 8458568, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 118992} +{"epoch": 1.01953125, "grad_norm": 0.1150798425078392, "learning_rate": 5.7466236393263005e-05, "loss": 0.0012194234877824783, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00122, "step": 261, "tokens/total": 8490840, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 119438} +{"epoch": 1.0234375, "grad_norm": 0.9463033676147461, "learning_rate": 5.717633247526522e-05, "loss": 0.0035801364574581385, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00359, "step": 262, "tokens/total": 8521792, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 119881} +{"epoch": 1.02734375, "grad_norm": 0.04941606894135475, "learning_rate": 5.688633799118971e-05, "loss": 0.0002292561694048345, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00023, "step": 263, "tokens/total": 8551336, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 120285} +{"epoch": 1.03125, "grad_norm": 0.044155512005090714, "learning_rate": 5.659626500889066e-05, "loss": 0.0003303611883893609, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00033, "step": 264, "tokens/total": 8579872, "tokens/train_per_sec_per_gpu": 7.09, "tokens/trainable": 120755} +{"epoch": 1.03515625, "grad_norm": 0.5383449196815491, "learning_rate": 5.6306125599488905e-05, "loss": 0.008458334021270275, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00849, "step": 265, "tokens/total": 8608920, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 121194} +{"epoch": 1.0390625, "grad_norm": 0.254611074924469, "learning_rate": 5.601593183686955e-05, "loss": 0.004170048050582409, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00418, "step": 266, "tokens/total": 8643600, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 121670} +{"epoch": 1.04296875, "grad_norm": 0.02541721612215042, "learning_rate": 5.572569579717961e-05, "loss": 0.00034417997812852263, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00034, "step": 267, "tokens/total": 8676032, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 122142} +{"epoch": 1.046875, "grad_norm": 0.023149235174059868, "learning_rate": 5.543542955832538e-05, "loss": 0.0002191613457398489, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00022, "step": 268, "tokens/total": 8713240, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 122585} +{"epoch": 1.05078125, "grad_norm": 0.1552293300628662, "learning_rate": 5.514514519946986e-05, "loss": 0.003700298024341464, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00371, "step": 269, "tokens/total": 8751664, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 123091} +{"epoch": 1.0546875, "grad_norm": 0.1528056412935257, "learning_rate": 5.485485480053015e-05, "loss": 0.0015275017358362675, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00153, "step": 270, "tokens/total": 8786112, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 123505} +{"epoch": 1.05859375, "grad_norm": 0.014945704489946365, "learning_rate": 5.4564570441674645e-05, "loss": 0.00014575656678061932, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00015, "step": 271, "tokens/total": 8816920, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 123953} +{"epoch": 1.0625, "grad_norm": 0.15859313309192657, "learning_rate": 5.42743042028204e-05, "loss": 0.0007236822275444865, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00072, "step": 272, "tokens/total": 8851408, "tokens/train_per_sec_per_gpu": 6.86, "tokens/trainable": 124400} +{"epoch": 1.06640625, "grad_norm": 0.03359290957450867, "learning_rate": 5.3984068163130464e-05, "loss": 0.0005632252432405949, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00056, "step": 273, "tokens/total": 8883768, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 124870} +{"epoch": 1.0703125, "grad_norm": 0.21117761731147766, "learning_rate": 5.369387440051111e-05, "loss": 0.0026441693771630526, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00265, "step": 274, "tokens/total": 8916200, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 125333} +{"epoch": 1.07421875, "grad_norm": 0.14327742159366608, "learning_rate": 5.340373499110935e-05, "loss": 0.001446493435651064, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00145, "step": 275, "tokens/total": 8949920, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 125834} +{"epoch": 1.078125, "grad_norm": 0.0504181794822216, "learning_rate": 5.3113662008810304e-05, "loss": 0.0003186087415087968, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00032, "step": 276, "tokens/total": 8986448, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 126316} +{"epoch": 1.08203125, "grad_norm": 0.2566686272621155, "learning_rate": 5.282366752473479e-05, "loss": 0.0009324439452029765, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00093, "step": 277, "tokens/total": 9020600, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 126798} +{"epoch": 1.0859375, "grad_norm": 0.011452681384980679, "learning_rate": 5.2533763606737005e-05, "loss": 0.00016118265921249986, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00016, "step": 278, "tokens/total": 9056040, "tokens/train_per_sec_per_gpu": 6.92, "tokens/trainable": 127223} +{"epoch": 1.08984375, "grad_norm": 0.33456701040267944, "learning_rate": 5.224396231890232e-05, "loss": 0.0024883903097361326, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00249, "step": 279, "tokens/total": 9087072, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 127672} +{"epoch": 1.09375, "grad_norm": 0.4698769152164459, "learning_rate": 5.195427572104522e-05, "loss": 0.0039928690530359745, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.004, "step": 280, "tokens/total": 9118376, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 128116} +{"epoch": 1.09765625, "grad_norm": 0.020618107169866562, "learning_rate": 5.166471586820751e-05, "loss": 0.00017071339243557304, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00017, "step": 281, "tokens/total": 9152704, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 128589} +{"epoch": 1.1015625, "grad_norm": 0.1111697256565094, "learning_rate": 5.1375294810156615e-05, "loss": 0.00040475113200955093, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.0004, "step": 282, "tokens/total": 9184448, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 129036} +{"epoch": 1.10546875, "grad_norm": 0.05592953786253929, "learning_rate": 5.1086024590884144e-05, "loss": 0.00029571595950983465, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.0003, "step": 283, "tokens/total": 9217392, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 129485} +{"epoch": 1.109375, "grad_norm": 0.02045895718038082, "learning_rate": 5.079691724810461e-05, "loss": 0.00011169625940965489, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00011, "step": 284, "tokens/total": 9252016, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 129920} +{"epoch": 1.11328125, "grad_norm": 0.30209437012672424, "learning_rate": 5.0507984812754684e-05, "loss": 0.0020230035297572613, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00203, "step": 285, "tokens/total": 9284008, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 130417} +{"epoch": 1.1171875, "grad_norm": 0.4084872007369995, "learning_rate": 5.021923930849237e-05, "loss": 0.0014215143164619803, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00142, "step": 286, "tokens/total": 9315984, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 130903} +{"epoch": 1.12109375, "grad_norm": 0.0014708992093801498, "learning_rate": 4.99306927511967e-05, "loss": 1.897069341794122e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00002, "step": 287, "tokens/total": 9347376, "tokens/train_per_sec_per_gpu": 10.49, "tokens/trainable": 131343} +{"epoch": 1.125, "grad_norm": 0.08228852599859238, "learning_rate": 4.964235714846775e-05, "loss": 0.0003117530432064086, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00031, "step": 288, "tokens/total": 9377424, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 131763} +{"epoch": 1.12890625, "grad_norm": 0.0013664372963830829, "learning_rate": 4.9354244499126866e-05, "loss": 1.91065228136722e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00002, "step": 289, "tokens/total": 9408104, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 132229} +{"epoch": 1.1328125, "grad_norm": 0.0011457388754934072, "learning_rate": 4.90663667927174e-05, "loss": 1.878062539617531e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00002, "step": 290, "tokens/total": 9439944, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 132678} +{"epoch": 1.13671875, "grad_norm": 0.013604179956018925, "learning_rate": 4.877873600900581e-05, "loss": 6.82277386658825e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00007, "step": 291, "tokens/total": 9472104, "tokens/train_per_sec_per_gpu": 6.26, "tokens/trainable": 133136} +{"epoch": 1.140625, "grad_norm": 0.006036225240677595, "learning_rate": 4.849136411748306e-05, "loss": 4.81364804727491e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00005, "step": 292, "tokens/total": 9508336, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 133608} +{"epoch": 1.14453125, "grad_norm": 0.3557804822921753, "learning_rate": 4.8204263076866574e-05, "loss": 0.0020529611501842737, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00206, "step": 293, "tokens/total": 9541552, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 134108} +{"epoch": 1.1484375, "grad_norm": 0.0004546220297925174, "learning_rate": 4.791744483460251e-05, "loss": 6.290975761658046e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00001, "step": 294, "tokens/total": 9575072, "tokens/train_per_sec_per_gpu": 6.63, "tokens/trainable": 134541} +{"epoch": 1.15234375, "grad_norm": 0.0022759949788451195, "learning_rate": 4.7630921326368736e-05, "loss": 1.9362916646059602e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00002, "step": 295, "tokens/total": 9610344, "tokens/train_per_sec_per_gpu": 6.46, "tokens/trainable": 134966} +{"epoch": 1.15625, "grad_norm": 0.056893058121204376, "learning_rate": 4.7344704475577916e-05, "loss": 0.00023780795163474977, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00024, "step": 296, "tokens/total": 9639728, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 135402} +{"epoch": 1.16015625, "grad_norm": 0.2275967299938202, "learning_rate": 4.705880619288153e-05, "loss": 0.0008881440735422075, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00089, "step": 297, "tokens/total": 9671424, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 135804} +{"epoch": 1.1640625, "grad_norm": 0.0015218616463243961, "learning_rate": 4.677323837567412e-05, "loss": 1.5122177501325496e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00002, "step": 298, "tokens/total": 9700808, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 136231} +{"epoch": 1.16796875, "grad_norm": 0.00048705239896662533, "learning_rate": 4.6488012907598146e-05, "loss": 7.021410965535324e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00001, "step": 299, "tokens/total": 9732312, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 136705} +{"epoch": 1.171875, "grad_norm": 0.023720895871520042, "learning_rate": 4.620314165804964e-05, "loss": 9.509408846497536e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.0001, "step": 300, "tokens/total": 9764624, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 137178} +{"epoch": 1.17578125, "grad_norm": 0.0073807197622954845, "learning_rate": 4.591863648168407e-05, "loss": 9.649947969592176e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00001, "step": 301, "tokens/total": 9798368, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 137643} +{"epoch": 1.1796875, "grad_norm": 0.0003350665792822838, "learning_rate": 4.5634509217923135e-05, "loss": 6.030918939359253e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00001, "step": 302, "tokens/total": 9830192, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 138078} +{"epoch": 1.18359375, "grad_norm": 0.0043416996486485004, "learning_rate": 4.535077169046201e-05, "loss": 2.989381755469367e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00003, "step": 303, "tokens/total": 9863112, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 138515} +{"epoch": 1.1875, "grad_norm": 0.009157489985227585, "learning_rate": 4.506743570677743e-05, "loss": 6.651488365605474e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00007, "step": 304, "tokens/total": 9893192, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 138948} +{"epoch": 1.19140625, "grad_norm": 0.4224410057067871, "learning_rate": 4.478451305763618e-05, "loss": 0.002108908724039793, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00211, "step": 305, "tokens/total": 9925712, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 139408} +{"epoch": 1.1953125, "grad_norm": 0.13366764783859253, "learning_rate": 4.450201551660454e-05, "loss": 0.000529408163856715, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00053, "step": 306, "tokens/total": 9955736, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 139840} +{"epoch": 1.19921875, "grad_norm": 0.7022703886032104, "learning_rate": 4.4219954839558276e-05, "loss": 0.0021079806610941887, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00211, "step": 307, "tokens/total": 9987160, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 140281} +{"epoch": 1.203125, "grad_norm": 3.046691656112671, "learning_rate": 4.393834276419352e-05, "loss": 0.012740198522806168, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.01282, "step": 308, "tokens/total": 10017360, "tokens/train_per_sec_per_gpu": 11.21, "tokens/trainable": 140776} +{"epoch": 1.20703125, "grad_norm": 3.514922857284546, "learning_rate": 4.36571910095382e-05, "loss": 0.001211988739669323, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00121, "step": 309, "tokens/total": 10052056, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 141228} +{"epoch": 1.2109375, "grad_norm": 0.02025669254362583, "learning_rate": 4.337651127546448e-05, "loss": 5.598886855295859e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00006, "step": 310, "tokens/total": 10085424, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 141679} +{"epoch": 1.21484375, "grad_norm": 1.3065648078918457, "learning_rate": 4.3096315242201736e-05, "loss": 0.012356710620224476, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.01243, "step": 311, "tokens/total": 10116824, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 142122} +{"epoch": 1.21875, "grad_norm": 0.821799099445343, "learning_rate": 4.2816614569850635e-05, "loss": 0.0021846715826541185, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00219, "step": 312, "tokens/total": 10150376, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 142561} +{"epoch": 1.22265625, "grad_norm": 0.08843373507261276, "learning_rate": 4.2537420897897864e-05, "loss": 0.00029405433451756835, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00029, "step": 313, "tokens/total": 10184432, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 143046} +{"epoch": 1.2265625, "grad_norm": 0.00028021406615152955, "learning_rate": 4.225874584473174e-05, "loss": 7.333140274567995e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00001, "step": 314, "tokens/total": 10214848, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 143493} +{"epoch": 1.23046875, "grad_norm": 0.12694819271564484, "learning_rate": 4.19806010071587e-05, "loss": 0.0001901122450362891, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00019, "step": 315, "tokens/total": 10247952, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 143956} +{"epoch": 1.234375, "grad_norm": 0.1051749661564827, "learning_rate": 4.170299795992081e-05, "loss": 0.0002563842572271824, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00026, "step": 316, "tokens/total": 10280648, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 144411} +{"epoch": 1.23828125, "grad_norm": 0.05007008835673332, "learning_rate": 4.142594825521398e-05, "loss": 0.00010933679004665464, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00011, "step": 317, "tokens/total": 10317712, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 144892} +{"epoch": 1.2421875, "grad_norm": 1.8343068361282349, "learning_rate": 4.114946342220728e-05, "loss": 0.004363153129816055, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00437, "step": 318, "tokens/total": 10351632, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 145339} +{"epoch": 1.24609375, "grad_norm": 0.2739590108394623, "learning_rate": 4.087355496656321e-05, "loss": 0.0011968470644205809, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.0012, "step": 319, "tokens/total": 10383336, "tokens/train_per_sec_per_gpu": 10.22, "tokens/trainable": 145811} +{"epoch": 1.25, "grad_norm": 0.5156742334365845, "learning_rate": 4.05982343699588e-05, "loss": 0.005560220684856176, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00558, "step": 320, "tokens/total": 10418016, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 146314} +{"epoch": 1.25390625, "grad_norm": 0.10872579365968704, "learning_rate": 4.0323513089607876e-05, "loss": 0.00022296421229839325, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00022, "step": 321, "tokens/total": 10451600, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 146781} +{"epoch": 1.2578125, "grad_norm": 0.0011327271349728107, "learning_rate": 4.004940255778431e-05, "loss": 1.4637299500463996e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00001, "step": 322, "tokens/total": 10487568, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 147225} +{"epoch": 1.26171875, "grad_norm": 0.019188078120350838, "learning_rate": 3.977591418134619e-05, "loss": 0.00013255448720883578, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00013, "step": 323, "tokens/total": 10520976, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 147673} +{"epoch": 1.265625, "grad_norm": 0.00047843673382885754, "learning_rate": 3.95030593412612e-05, "loss": 8.22926995169837e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00001, "step": 324, "tokens/total": 10552776, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 148103} +{"epoch": 1.26953125, "grad_norm": 0.018736667931079865, "learning_rate": 3.923084939213296e-05, "loss": 1.9631488612503745e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00002, "step": 325, "tokens/total": 10582576, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 148535} +{"epoch": 1.2734375, "grad_norm": 0.26333087682724, "learning_rate": 3.895929566172861e-05, "loss": 0.0008602222660556436, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00086, "step": 326, "tokens/total": 10615600, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 148999} +{"epoch": 1.27734375, "grad_norm": 0.015485723502933979, "learning_rate": 3.868840945050728e-05, "loss": 4.957894634571858e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00005, "step": 327, "tokens/total": 10650688, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 149431} +{"epoch": 1.28125, "grad_norm": 0.0010432489216327667, "learning_rate": 3.841820203114995e-05, "loss": 1.0838163689186331e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00001, "step": 328, "tokens/total": 10681816, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 149886} +{"epoch": 1.28515625, "grad_norm": 0.001272315625101328, "learning_rate": 3.814868464809027e-05, "loss": 8.824039468890987e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00001, "step": 329, "tokens/total": 10711728, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 150288} +{"epoch": 1.2890625, "grad_norm": 0.6803052425384521, "learning_rate": 3.787986851704667e-05, "loss": 0.005081913899630308, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00509, "step": 330, "tokens/total": 10744952, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 150733} +{"epoch": 1.29296875, "grad_norm": 0.08443129062652588, "learning_rate": 3.7611764824555654e-05, "loss": 0.0002421980316285044, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00024, "step": 331, "tokens/total": 10776192, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 151207} +{"epoch": 1.296875, "grad_norm": 0.04822001978754997, "learning_rate": 3.734438472750619e-05, "loss": 0.00022705337323714048, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00023, "step": 332, "tokens/total": 10813336, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 151694} +{"epoch": 1.30078125, "grad_norm": 0.010836242698132992, "learning_rate": 3.707773935267552e-05, "loss": 2.304869303770829e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00002, "step": 333, "tokens/total": 10847808, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 152188} +{"epoch": 1.3046875, "grad_norm": 0.02263072319328785, "learning_rate": 3.68118397962661e-05, "loss": 4.6144912630552426e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00005, "step": 334, "tokens/total": 10879896, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 152596} +{"epoch": 1.30859375, "grad_norm": 0.20843619108200073, "learning_rate": 3.654669712344384e-05, "loss": 0.0002741016214713454, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00027, "step": 335, "tokens/total": 10908976, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 153052} +{"epoch": 1.3125, "grad_norm": 0.003301863092929125, "learning_rate": 3.628232236787763e-05, "loss": 1.4104457477515098e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00001, "step": 336, "tokens/total": 10943152, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 153491} +{"epoch": 1.31640625, "grad_norm": 0.11820019781589508, "learning_rate": 3.6018726531280144e-05, "loss": 0.00040426637860946357, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.0004, "step": 337, "tokens/total": 10978184, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 153983} +{"epoch": 1.3203125, "grad_norm": 0.01535138301551342, "learning_rate": 3.575592058295017e-05, "loss": 6.893488171044737e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00007, "step": 338, "tokens/total": 11010056, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 154464} +{"epoch": 1.32421875, "grad_norm": 0.0010633602505549788, "learning_rate": 3.549391545931585e-05, "loss": 1.0710173228289932e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00001, "step": 339, "tokens/total": 11040968, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 154924} +{"epoch": 1.328125, "grad_norm": 0.5236793160438538, "learning_rate": 3.5232722063479914e-05, "loss": 0.006533716805279255, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00656, "step": 340, "tokens/total": 11073424, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 155373} +{"epoch": 1.33203125, "grad_norm": 0.023613059893250465, "learning_rate": 3.49723512647657e-05, "loss": 0.00010899423796217889, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00011, "step": 341, "tokens/total": 11109224, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 155873} +{"epoch": 1.3359375, "grad_norm": 0.6122479438781738, "learning_rate": 3.471281389826491e-05, "loss": 0.006281656213104725, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.0063, "step": 342, "tokens/total": 11139856, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 156278} +{"epoch": 1.33984375, "grad_norm": 0.007751265075057745, "learning_rate": 3.4454120764386764e-05, "loss": 4.405496656545438e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00004, "step": 343, "tokens/total": 11175224, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 156733} +{"epoch": 1.34375, "grad_norm": 0.49642691016197205, "learning_rate": 3.4196282628408526e-05, "loss": 0.012115669436752796, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01219, "step": 344, "tokens/total": 11207112, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 157203} +{"epoch": 1.34765625, "grad_norm": 0.11773742735385895, "learning_rate": 3.3939310220027456e-05, "loss": 0.000792464183177799, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00079, "step": 345, "tokens/total": 11240432, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 157707} +{"epoch": 1.3515625, "grad_norm": 0.40878826379776, "learning_rate": 3.3683214232914404e-05, "loss": 0.0024362285621464252, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00244, "step": 346, "tokens/total": 11274584, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 158180} +{"epoch": 1.35546875, "grad_norm": 1.2314984798431396, "learning_rate": 3.342800532426873e-05, "loss": 0.0090407095849514, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00908, "step": 347, "tokens/total": 11307376, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 158646} +{"epoch": 1.359375, "grad_norm": 0.1840517669916153, "learning_rate": 3.317369411437484e-05, "loss": 0.000754146312829107, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00075, "step": 348, "tokens/total": 11340640, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 159115} +{"epoch": 1.36328125, "grad_norm": 0.15070143342018127, "learning_rate": 3.292029118616024e-05, "loss": 0.00010950530122499913, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00011, "step": 349, "tokens/total": 11371632, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 159538} +{"epoch": 1.3671875, "grad_norm": 0.009912174195051193, "learning_rate": 3.266780708475511e-05, "loss": 5.5585987865924835e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00006, "step": 350, "tokens/total": 11404248, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 160016} +{"epoch": 1.37109375, "grad_norm": 0.3307338058948517, "learning_rate": 3.241625231705354e-05, "loss": 0.001001198892481625, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.001, "step": 351, "tokens/total": 11439552, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 160475} +{"epoch": 1.375, "grad_norm": 0.28541165590286255, "learning_rate": 3.216563735127618e-05, "loss": 0.0011998440604656935, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.0012, "step": 352, "tokens/total": 11470944, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 160952} +{"epoch": 1.37890625, "grad_norm": 0.08427648991346359, "learning_rate": 3.191597261653475e-05, "loss": 0.00055139564210549, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00055, "step": 353, "tokens/total": 11506304, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 161420} +{"epoch": 1.3828125, "grad_norm": 0.26355308294296265, "learning_rate": 3.166726850239794e-05, "loss": 0.0003640790528152138, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00036, "step": 354, "tokens/total": 11535424, "tokens/train_per_sec_per_gpu": 11.07, "tokens/trainable": 161868} +{"epoch": 1.38671875, "grad_norm": 0.10192650556564331, "learning_rate": 3.141953535845912e-05, "loss": 0.0001605010766070336, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00016, "step": 355, "tokens/total": 11565176, "tokens/train_per_sec_per_gpu": 10.34, "tokens/trainable": 162318} +{"epoch": 1.390625, "grad_norm": 0.09554077684879303, "learning_rate": 3.11727834939056e-05, "loss": 0.0009927380597218871, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00099, "step": 356, "tokens/total": 11595528, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 162789} +{"epoch": 1.39453125, "grad_norm": 0.14256596565246582, "learning_rate": 3.092702317708967e-05, "loss": 0.001818044576793909, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00182, "step": 357, "tokens/total": 11627248, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 163254} +{"epoch": 1.3984375, "grad_norm": 0.02948250249028206, "learning_rate": 3.0682264635101276e-05, "loss": 2.984954153362196e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00003, "step": 358, "tokens/total": 11659440, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 163715} +{"epoch": 1.40234375, "grad_norm": 0.11227059364318848, "learning_rate": 3.0438518053342407e-05, "loss": 0.0009626607643440366, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00096, "step": 359, "tokens/total": 11693288, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 164197} +{"epoch": 1.40625, "grad_norm": 0.07695257663726807, "learning_rate": 3.0195793575103266e-05, "loss": 0.000768988044001162, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00077, "step": 360, "tokens/total": 11730888, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 164661} +{"epoch": 1.41015625, "grad_norm": 0.5442395806312561, "learning_rate": 2.9954101301140146e-05, "loss": 0.005780606530606747, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.0058, "step": 361, "tokens/total": 11765656, "tokens/train_per_sec_per_gpu": 6.96, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 2.977313756942749, "learning_rate": 2.9713451289255123e-05, "loss": 0.001971584977582097, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00197, "step": 362, "tokens/total": 11793456, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 165552} +{"epoch": 1.41796875, "grad_norm": 0.0649840384721756, "learning_rate": 2.9473853553877484e-05, "loss": 0.0002524603623896837, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00025, "step": 363, "tokens/total": 11826280, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 166026} +{"epoch": 1.421875, "grad_norm": 0.12338749319314957, "learning_rate": 2.9235318065647e-05, "loss": 0.0006315425271168351, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00063, "step": 364, "tokens/total": 11863552, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 166486} +{"epoch": 1.42578125, "grad_norm": 0.046615537256002426, "learning_rate": 2.8997854750998964e-05, "loss": 0.00026118889218196273, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00026, "step": 365, "tokens/total": 11898616, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 166959} +{"epoch": 1.4296875, "grad_norm": 0.8705919981002808, "learning_rate": 2.8761473491751258e-05, "loss": 0.002134400187060237, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00214, "step": 366, "tokens/total": 11930728, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 167394} +{"epoch": 1.43359375, "grad_norm": 0.02422679215669632, "learning_rate": 2.8526184124692883e-05, "loss": 0.00017122726421803236, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00017, "step": 367, "tokens/total": 11964976, "tokens/train_per_sec_per_gpu": 6.45, "tokens/trainable": 167832} +{"epoch": 1.4375, "grad_norm": 0.014415659010410309, "learning_rate": 2.829199644117484e-05, "loss": 4.801769318873994e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00005, "step": 368, "tokens/total": 11996696, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 168242} +{"epoch": 1.44140625, "grad_norm": 0.054681938141584396, "learning_rate": 2.8058920186702553e-05, "loss": 0.00033841311233118176, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00034, "step": 369, "tokens/total": 12033432, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 168715} +{"epoch": 1.4453125, "grad_norm": 0.35256192088127136, "learning_rate": 2.782696506053033e-05, "loss": 0.002689959481358528, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00269, "step": 370, "tokens/total": 12067616, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 169147} +{"epoch": 1.44921875, "grad_norm": 0.32176223397254944, "learning_rate": 2.7596140715257824e-05, "loss": 0.001326177385635674, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00133, "step": 371, "tokens/total": 12099008, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 169628} +{"epoch": 1.453125, "grad_norm": 0.0022943296935409307, "learning_rate": 2.7366456756428184e-05, "loss": 1.6666836017975584e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00002, "step": 372, "tokens/total": 12131304, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 170085} +{"epoch": 1.45703125, "grad_norm": 0.07394909113645554, "learning_rate": 2.7137922742128486e-05, "loss": 0.0002918758546002209, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00029, "step": 373, "tokens/total": 12164104, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 170584} +{"epoch": 1.4609375, "grad_norm": 0.7528867125511169, "learning_rate": 2.691054818259188e-05, "loss": 0.011382165364921093, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.01145, "step": 374, "tokens/total": 12197776, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 171058} +{"epoch": 1.46484375, "grad_norm": 0.08581449091434479, "learning_rate": 2.6684342539801933e-05, "loss": 0.0004928004345856607, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00049, "step": 375, "tokens/total": 12228064, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 171518} +{"epoch": 1.46875, "grad_norm": 0.17470915615558624, "learning_rate": 2.645931522709877e-05, "loss": 0.0018082462484017015, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00181, "step": 376, "tokens/total": 12261688, "tokens/train_per_sec_per_gpu": 10.83, "tokens/trainable": 172007} +{"epoch": 1.47265625, "grad_norm": 0.15729019045829773, "learning_rate": 2.6235475608787365e-05, "loss": 0.001184638007543981, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00119, "step": 377, "tokens/total": 12294240, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 172468} +{"epoch": 1.4765625, "grad_norm": 0.04677904024720192, "learning_rate": 2.6012832999747916e-05, "loss": 0.00042954349191859365, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00043, "step": 378, "tokens/total": 12324528, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 172923} +{"epoch": 1.48046875, "grad_norm": 0.380440354347229, "learning_rate": 2.579139666504821e-05, "loss": 0.01837894693017006, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01855, "step": 379, "tokens/total": 12360288, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 173370} +{"epoch": 1.484375, "grad_norm": 0.018091153353452682, "learning_rate": 2.557117581955798e-05, "loss": 0.00011266102956142277, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00011, "step": 380, "tokens/total": 12391216, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 173837} +{"epoch": 1.48828125, "grad_norm": 0.0028983347583562136, "learning_rate": 2.5352179627565532e-05, "loss": 2.6217507183901034e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00003, "step": 381, "tokens/total": 12423624, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 174294} +{"epoch": 1.4921875, "grad_norm": 0.006486387457698584, "learning_rate": 2.5134417202396277e-05, "loss": 2.7570933525566943e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00003, "step": 382, "tokens/total": 12456032, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 174701} +{"epoch": 1.49609375, "grad_norm": 0.029854614287614822, "learning_rate": 2.491789760603361e-05, "loss": 0.00019594014156609774, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.0002, "step": 383, "tokens/total": 12489336, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 175131} +{"epoch": 1.5, "grad_norm": 0.011349949985742569, "learning_rate": 2.4702629848741764e-05, "loss": 4.69539372716099e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00005, "step": 384, "tokens/total": 12521944, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 175586} +{"epoch": 1.50390625, "grad_norm": 0.23185883462429047, "learning_rate": 2.4488622888690785e-05, "loss": 0.002170766703784466, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00217, "step": 385, "tokens/total": 12554024, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 176026} +{"epoch": 1.5078125, "grad_norm": 0.3366059958934784, "learning_rate": 2.427588563158384e-05, "loss": 0.008860510773956776, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.0089, "step": 386, "tokens/total": 12587104, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 176512} +{"epoch": 1.51171875, "grad_norm": 0.22657103836536407, "learning_rate": 2.406442693028651e-05, "loss": 0.0013160828966647387, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00132, "step": 387, "tokens/total": 12617456, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 176943} +{"epoch": 1.515625, "grad_norm": 0.007046757731586695, "learning_rate": 2.3854255584458547e-05, "loss": 4.303145396988839e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00004, "step": 388, "tokens/total": 12652696, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 177370} +{"epoch": 1.51953125, "grad_norm": 0.009818063117563725, "learning_rate": 2.3645380340187508e-05, "loss": 7.749958604108542e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00008, "step": 389, "tokens/total": 12685792, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 177851} +{"epoch": 1.5234375, "grad_norm": 0.13789153099060059, "learning_rate": 2.3437809889624914e-05, "loss": 0.0016887710662558675, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00169, "step": 390, "tokens/total": 12716120, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 178310} +{"epoch": 1.52734375, "grad_norm": 0.04858309403061867, "learning_rate": 2.3231552870624487e-05, "loss": 0.00031821688753552735, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00032, "step": 391, "tokens/total": 12746216, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 178736} +{"epoch": 1.53125, "grad_norm": 0.007686348631978035, "learning_rate": 2.3026617866382657e-05, "loss": 9.757574298419058e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.0001, "step": 392, "tokens/total": 12778592, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 179233} +{"epoch": 1.53515625, "grad_norm": 0.084551140666008, "learning_rate": 2.2823013405081507e-05, "loss": 0.000442837190348655, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00044, "step": 393, "tokens/total": 12812008, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 179730} +{"epoch": 1.5390625, "grad_norm": 0.04248236492276192, "learning_rate": 2.2620747959533722e-05, "loss": 0.0001561084936838597, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00016, "step": 394, "tokens/total": 12844264, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 180227} +{"epoch": 1.54296875, "grad_norm": 0.01261988840997219, "learning_rate": 2.2419829946830123e-05, "loss": 8.436036296188831e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00008, "step": 395, "tokens/total": 12878328, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 180687} +{"epoch": 1.546875, "grad_norm": 0.14395488798618317, "learning_rate": 2.2220267727989325e-05, "loss": 0.00100702082272619, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00101, "step": 396, "tokens/total": 12907912, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 181141} +{"epoch": 1.55078125, "grad_norm": 0.1286955028772354, "learning_rate": 2.202206960760984e-05, "loss": 0.0018133112462237477, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00181, "step": 397, "tokens/total": 12941968, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 181648} +{"epoch": 1.5546875, "grad_norm": 0.011072383262217045, "learning_rate": 2.182524383352446e-05, "loss": 0.0001285920588998124, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00013, "step": 398, "tokens/total": 12975240, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 182109} +{"epoch": 1.55859375, "grad_norm": 0.2873384952545166, "learning_rate": 2.1629798596457056e-05, "loss": 0.005032851826399565, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00505, "step": 399, "tokens/total": 13006912, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 182559} +{"epoch": 1.5625, "grad_norm": 0.06345149874687195, "learning_rate": 2.1435742029681725e-05, "loss": 0.00032752996776252985, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00033, "step": 400, "tokens/total": 13037640, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 182985} +{"epoch": 1.56640625, "grad_norm": 0.012339959852397442, "learning_rate": 2.124308220868431e-05, "loss": 6.854987441329286e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00007, "step": 401, "tokens/total": 13070608, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 183440} +{"epoch": 1.5703125, "grad_norm": 0.05786518380045891, "learning_rate": 2.105182715082638e-05, "loss": 0.0007114062900654972, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00071, "step": 402, "tokens/total": 13101784, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 183917} +{"epoch": 1.57421875, "grad_norm": 1.1884841918945312, "learning_rate": 2.0861984815011552e-05, "loss": 0.01135345920920372, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.01142, "step": 403, "tokens/total": 13133528, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 184348} +{"epoch": 1.578125, "grad_norm": 0.019514918327331543, "learning_rate": 2.0673563101354323e-05, "loss": 0.00016828802472446114, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00017, "step": 404, "tokens/total": 13167088, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 184786} +{"epoch": 1.58203125, "grad_norm": 0.15433049201965332, "learning_rate": 2.0486569850851317e-05, "loss": 0.0019203252159059048, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00192, "step": 405, "tokens/total": 13200392, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 185249} +{"epoch": 1.5859375, "grad_norm": 0.025796880945563316, "learning_rate": 2.0301012845054956e-05, "loss": 0.00023781356867402792, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00024, "step": 406, "tokens/total": 13235488, "tokens/train_per_sec_per_gpu": 6.92, "tokens/trainable": 185680} +{"epoch": 1.58984375, "grad_norm": 0.5973268151283264, "learning_rate": 2.011689980574966e-05, "loss": 0.005254978314042091, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00527, "step": 407, "tokens/total": 13269416, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 186100} +{"epoch": 1.59375, "grad_norm": 0.07131307572126389, "learning_rate": 1.993423839463052e-05, "loss": 0.0005437479121610522, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00054, "step": 408, "tokens/total": 13301360, "tokens/train_per_sec_per_gpu": 10.48, "tokens/trainable": 186565} +{"epoch": 1.59765625, "grad_norm": 0.015336783602833748, "learning_rate": 1.975303621298445e-05, "loss": 0.00014950038166716695, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00015, "step": 409, "tokens/total": 13336392, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 187017} +{"epoch": 1.6015625, "grad_norm": 0.005379797425121069, "learning_rate": 1.957330080137385e-05, "loss": 5.477289960253984e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00005, "step": 410, "tokens/total": 13366072, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 187468} +{"epoch": 1.60546875, "grad_norm": 0.037266071885824203, "learning_rate": 1.9395039639322864e-05, "loss": 0.00030950148357078433, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00031, "step": 411, "tokens/total": 13400832, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 187961} +{"epoch": 1.609375, "grad_norm": 0.007206721697002649, "learning_rate": 1.9218260145006073e-05, "loss": 4.160354728810489e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00004, "step": 412, "tokens/total": 13434920, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 188447} +{"epoch": 1.61328125, "grad_norm": 0.009946510195732117, "learning_rate": 1.904296967493982e-05, "loss": 0.0001081224181689322, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00011, "step": 413, "tokens/total": 13466512, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 188903} +{"epoch": 1.6171875, "grad_norm": 0.055324286222457886, "learning_rate": 1.8869175523676064e-05, "loss": 0.00047871519927866757, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00048, "step": 414, "tokens/total": 13500352, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 189398} +{"epoch": 1.62109375, "grad_norm": 0.11924748867750168, "learning_rate": 1.869688492349885e-05, "loss": 0.001002641161903739, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.001, "step": 415, "tokens/total": 13532448, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 189850} +{"epoch": 1.625, "grad_norm": 0.04723189398646355, "learning_rate": 1.85261050441233e-05, "loss": 0.0004015158920083195, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.0004, "step": 416, "tokens/total": 13565648, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 190306} +{"epoch": 1.62890625, "grad_norm": 0.2504064440727234, "learning_rate": 1.8356842992397304e-05, "loss": 0.002303785178810358, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00231, "step": 417, "tokens/total": 13597600, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 190746} +{"epoch": 1.6328125, "grad_norm": 0.29677650332450867, "learning_rate": 1.8189105812005714e-05, "loss": 0.00151724461466074, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00152, "step": 418, "tokens/total": 13630456, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 191156} +{"epoch": 1.63671875, "grad_norm": 0.021297171711921692, "learning_rate": 1.802290048317732e-05, "loss": 0.0002958310942631215, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.0003, "step": 419, "tokens/total": 13660896, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 191573} +{"epoch": 1.640625, "grad_norm": 0.29345276951789856, "learning_rate": 1.785823392239424e-05, "loss": 0.0030892007052898407, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00309, "step": 420, "tokens/total": 13693176, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 192014} +{"epoch": 1.64453125, "grad_norm": 0.048612210899591446, "learning_rate": 1.7695112982104225e-05, "loss": 0.00024492695229128003, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00024, "step": 421, "tokens/total": 13725400, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 192453} +{"epoch": 1.6484375, "grad_norm": 0.09738782048225403, "learning_rate": 1.7533544450435433e-05, "loss": 0.0006403346196748316, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00064, "step": 422, "tokens/total": 13752752, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 192886} +{"epoch": 1.65234375, "grad_norm": 0.0018251618603244424, "learning_rate": 1.7373535050913946e-05, "loss": 2.5675708457129076e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00003, "step": 423, "tokens/total": 13786136, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 193323} +{"epoch": 1.65625, "grad_norm": 2.279996871948242, "learning_rate": 1.721509144218405e-05, "loss": 0.0013278209371492267, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00133, "step": 424, "tokens/total": 13815904, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 193753} +{"epoch": 1.66015625, "grad_norm": 0.02878432907164097, "learning_rate": 1.705822021773101e-05, "loss": 0.00030436518136411905, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.0003, "step": 425, "tokens/total": 13847976, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 194240} +{"epoch": 1.6640625, "grad_norm": 0.01931927166879177, "learning_rate": 1.69029279056068e-05, "loss": 0.00026718113804236054, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00027, "step": 426, "tokens/total": 13883200, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 194680} +{"epoch": 1.66796875, "grad_norm": 0.04420853778719902, "learning_rate": 1.6749220968158415e-05, "loss": 0.00018104841001331806, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00018, "step": 427, "tokens/total": 13916080, "tokens/train_per_sec_per_gpu": 10.43, "tokens/trainable": 195186} +{"epoch": 1.671875, "grad_norm": 0.0619470588862896, "learning_rate": 1.659710580175893e-05, "loss": 0.0007514787721447647, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00075, "step": 428, "tokens/total": 13946448, "tokens/train_per_sec_per_gpu": 10.19, "tokens/trainable": 195693} +{"epoch": 1.67578125, "grad_norm": 0.1014176681637764, "learning_rate": 1.644658873654133e-05, "loss": 0.0008918903768062592, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00089, "step": 429, "tokens/total": 13975696, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 196130} +{"epoch": 1.6796875, "grad_norm": 0.007490174844861031, "learning_rate": 1.629767603613508e-05, "loss": 9.545023931423202e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.0001, "step": 430, "tokens/total": 14002616, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 196530} +{"epoch": 1.68359375, "grad_norm": 0.01692194864153862, "learning_rate": 1.615037389740547e-05, "loss": 0.0003162719658575952, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00032, "step": 431, "tokens/total": 14034848, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 196974} +{"epoch": 1.6875, "grad_norm": 0.32038193941116333, "learning_rate": 1.600468845019576e-05, "loss": 0.0005938211688771844, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00059, "step": 432, "tokens/total": 14069032, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 197454} +{"epoch": 1.69140625, "grad_norm": 0.0010429027024656534, "learning_rate": 1.5860625757072092e-05, "loss": 1.8767161236610264e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00002, "step": 433, "tokens/total": 14101296, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 197902} +{"epoch": 1.6953125, "grad_norm": 0.05182128772139549, "learning_rate": 1.571819181307116e-05, "loss": 0.0005632045213133097, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00056, "step": 434, "tokens/total": 14135880, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 198374} +{"epoch": 1.69921875, "grad_norm": 0.017138037830591202, "learning_rate": 1.557739254545075e-05, "loss": 0.00017759925685822964, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00018, "step": 435, "tokens/total": 14166040, "tokens/train_per_sec_per_gpu": 10.71, "tokens/trainable": 198846} +{"epoch": 1.703125, "grad_norm": 0.06882758438587189, "learning_rate": 1.543823381344311e-05, "loss": 0.0010052649304270744, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00101, "step": 436, "tokens/total": 14196032, "tokens/train_per_sec_per_gpu": 10.82, "tokens/trainable": 199320} +{"epoch": 1.70703125, "grad_norm": 0.1469891369342804, "learning_rate": 1.5300721408011114e-05, "loss": 0.0002418892690911889, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00024, "step": 437, "tokens/total": 14228008, "tokens/train_per_sec_per_gpu": 10.3, "tokens/trainable": 199777} +{"epoch": 1.7109375, "grad_norm": 0.0033605352509766817, "learning_rate": 1.5164861051607254e-05, "loss": 3.7228794099064544e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.76, "memory/max_allocated (GiB)": 65.76, "ppl": 1.00004, "step": 438, "tokens/total": 14261744, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 200194} +{"epoch": 1.71484375, "grad_norm": 0.00569489598274231, "learning_rate": 1.5030658397935521e-05, "loss": 6.569838296854869e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00007, "step": 439, "tokens/total": 14295208, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 200663} +{"epoch": 1.71875, "grad_norm": 0.019205676391720772, "learning_rate": 1.4898119031716104e-05, "loss": 7.179766544140875e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00007, "step": 440, "tokens/total": 14327264, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 201121} +{"epoch": 1.72265625, "grad_norm": 0.0023226507473737, "learning_rate": 1.476724846845306e-05, "loss": 2.8607781132450327e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00003, "step": 441, "tokens/total": 14363088, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 201598} +{"epoch": 1.7265625, "grad_norm": 0.026453014463186264, "learning_rate": 1.463805215420471e-05, "loss": 0.00017156917601823807, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00017, "step": 442, "tokens/total": 14396088, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 202068} +{"epoch": 1.73046875, "grad_norm": 0.021105367690324783, "learning_rate": 1.451053546535705e-05, "loss": 0.00011699977767420933, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00012, "step": 443, "tokens/total": 14429232, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 202524} +{"epoch": 1.734375, "grad_norm": 0.04233882948756218, "learning_rate": 1.438470370840001e-05, "loss": 0.00017683242913335562, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00018, "step": 444, "tokens/total": 14463264, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 202996} +{"epoch": 1.73828125, "grad_norm": 0.0010658823885023594, "learning_rate": 1.4260562119706606e-05, "loss": 1.5705010810052045e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00002, "step": 445, "tokens/total": 14493552, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 203470} +{"epoch": 1.7421875, "grad_norm": 0.0033716727048158646, "learning_rate": 1.413811586531508e-05, "loss": 3.678793291328475e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00004, "step": 446, "tokens/total": 14525240, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 203919} +{"epoch": 1.74609375, "grad_norm": 0.055613141506910324, "learning_rate": 1.4017370040713884e-05, "loss": 0.0004297153791412711, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00043, "step": 447, "tokens/total": 14559560, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 204376} +{"epoch": 1.75, "grad_norm": 0.02910628169775009, "learning_rate": 1.3898329670629645e-05, "loss": 0.00022421963512897491, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00022, "step": 448, "tokens/total": 14594160, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 204821} +{"epoch": 1.75390625, "grad_norm": 0.0007711257203482091, "learning_rate": 1.3780999708818058e-05, "loss": 1.3441291230265051e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00001, "step": 449, "tokens/total": 14624224, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 205271} +{"epoch": 1.7578125, "grad_norm": 0.004567094147205353, "learning_rate": 1.3665385037857758e-05, "loss": 3.474264303804375e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00003, "step": 450, "tokens/total": 14654984, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 205723} +{"epoch": 1.76171875, "grad_norm": 0.006839292123913765, "learning_rate": 1.3551490468947126e-05, "loss": 7.377313886536285e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00007, "step": 451, "tokens/total": 14685440, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 206169} +{"epoch": 1.765625, "grad_norm": 0.0444706529378891, "learning_rate": 1.3439320741704075e-05, "loss": 0.00019871533731929958, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.0002, "step": 452, "tokens/total": 14716120, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 206592} +{"epoch": 1.76953125, "grad_norm": 0.12053992599248886, "learning_rate": 1.3328880523968808e-05, "loss": 0.00026420794893056154, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00026, "step": 453, "tokens/total": 14748424, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 207055} +{"epoch": 1.7734375, "grad_norm": 0.27200424671173096, "learning_rate": 1.3220174411609587e-05, "loss": 0.00200660596601665, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00201, "step": 454, "tokens/total": 14781256, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 207494} +{"epoch": 1.77734375, "grad_norm": 0.0052926200442016125, "learning_rate": 1.3113206928331471e-05, "loss": 4.704310049419291e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00005, "step": 455, "tokens/total": 14809552, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 207956} +{"epoch": 1.78125, "grad_norm": 0.008776476606726646, "learning_rate": 1.300798252548806e-05, "loss": 8.274190622614697e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00008, "step": 456, "tokens/total": 14841808, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 208408} +{"epoch": 1.78515625, "grad_norm": 0.0461672805249691, "learning_rate": 1.2904505581896265e-05, "loss": 0.00025538657791912556, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00026, "step": 457, "tokens/total": 14875896, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 208889} +{"epoch": 1.7890625, "grad_norm": 0.02836505137383938, "learning_rate": 1.2802780403654082e-05, "loss": 0.0002334640739718452, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00023, "step": 458, "tokens/total": 14908424, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 209336} +{"epoch": 1.79296875, "grad_norm": 0.07742038369178772, "learning_rate": 1.2702811223961408e-05, "loss": 0.0010890542762354016, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00109, "step": 459, "tokens/total": 14939168, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 209797} +{"epoch": 1.796875, "grad_norm": 0.018306920304894447, "learning_rate": 1.2604602202943861e-05, "loss": 5.818731369799934e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00006, "step": 460, "tokens/total": 14969872, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 210240} +{"epoch": 1.80078125, "grad_norm": 0.020897604525089264, "learning_rate": 1.2508157427479686e-05, "loss": 0.0002175606059608981, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00022, "step": 461, "tokens/total": 15001472, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 210702} +{"epoch": 1.8046875, "grad_norm": 0.15570445358753204, "learning_rate": 1.2413480911029655e-05, "loss": 0.0022189307492226362, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00222, "step": 462, "tokens/total": 15030592, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 211140} +{"epoch": 1.80859375, "grad_norm": 0.004757544957101345, "learning_rate": 1.2320576593470082e-05, "loss": 1.7064936400856823e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00002, "step": 463, "tokens/total": 15063648, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 211575} +{"epoch": 1.8125, "grad_norm": 0.07109804451465607, "learning_rate": 1.2229448340928828e-05, "loss": 0.0003374799562152475, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00034, "step": 464, "tokens/total": 15098248, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 212083} +{"epoch": 1.81640625, "grad_norm": 0.058545682579278946, "learning_rate": 1.2140099945624458e-05, "loss": 3.761678090086207e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00004, "step": 465, "tokens/total": 15133216, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 212558} +{"epoch": 1.8203125, "grad_norm": 0.020004600286483765, "learning_rate": 1.205253512570841e-05, "loss": 0.00018392690981272608, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00018, "step": 466, "tokens/total": 15165416, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 213014} +{"epoch": 1.82421875, "grad_norm": 0.00925489328801632, "learning_rate": 1.1966757525110255e-05, "loss": 7.707839540671557e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00008, "step": 467, "tokens/total": 15193688, "tokens/train_per_sec_per_gpu": 10.17, "tokens/trainable": 213465} +{"epoch": 1.828125, "grad_norm": 0.06753110885620117, "learning_rate": 1.1882770713386095e-05, "loss": 0.0007444887887686491, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00074, "step": 468, "tokens/total": 15228600, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 213940} +{"epoch": 1.83203125, "grad_norm": 0.03782886639237404, "learning_rate": 1.180057818556998e-05, "loss": 0.00019471056293696165, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00019, "step": 469, "tokens/total": 15262192, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 214415} +{"epoch": 1.8359375, "grad_norm": 0.00601538922637701, "learning_rate": 1.1720183362028494e-05, "loss": 2.9515515052480623e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00003, "step": 470, "tokens/total": 15294920, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 214880} +{"epoch": 1.83984375, "grad_norm": 0.01366535946726799, "learning_rate": 1.1641589588318387e-05, "loss": 6.891224620630965e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00007, "step": 471, "tokens/total": 15327432, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 215334} +{"epoch": 1.84375, "grad_norm": 0.0032548154704272747, "learning_rate": 1.1564800135047418e-05, "loss": 1.3692360880668275e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00001, "step": 472, "tokens/total": 15359928, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 215822} +{"epoch": 1.84765625, "grad_norm": 0.060404155403375626, "learning_rate": 1.148981819773816e-05, "loss": 0.0004941668594256043, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00049, "step": 473, "tokens/total": 15391400, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 216239} +{"epoch": 1.8515625, "grad_norm": 0.000830934732221067, "learning_rate": 1.1416646896695086e-05, "loss": 1.3226333976490423e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00001, "step": 474, "tokens/total": 15424144, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 216691} +{"epoch": 1.85546875, "grad_norm": 0.008953779935836792, "learning_rate": 1.1345289276874717e-05, "loss": 8.687042281962931e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00009, "step": 475, "tokens/total": 15457680, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 217162} +{"epoch": 1.859375, "grad_norm": 0.05256018042564392, "learning_rate": 1.1275748307758873e-05, "loss": 0.00024153823324013501, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00024, "step": 476, "tokens/total": 15492056, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 217638} +{"epoch": 1.86328125, "grad_norm": 0.012865480035543442, "learning_rate": 1.1208026883231147e-05, "loss": 5.3686606406699866e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00005, "step": 477, "tokens/total": 15521544, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 218053} +{"epoch": 1.8671875, "grad_norm": 0.009344838559627533, "learning_rate": 1.1142127821456433e-05, "loss": 9.713098552310839e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.0001, "step": 478, "tokens/total": 15553320, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 218503} +{"epoch": 1.87109375, "grad_norm": 0.034292496740818024, "learning_rate": 1.1078053864763674e-05, "loss": 7.222384738270193e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00007, "step": 479, "tokens/total": 15586208, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 218964} +{"epoch": 1.875, "grad_norm": 0.34432893991470337, "learning_rate": 1.1015807679531756e-05, "loss": 0.00018384968279860914, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00018, "step": 480, "tokens/total": 15616432, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 219412} +{"epoch": 1.87890625, "grad_norm": 0.0647139921784401, "learning_rate": 1.0955391856078528e-05, "loss": 0.0004043865483254194, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.0004, "step": 481, "tokens/total": 15650056, "tokens/train_per_sec_per_gpu": 10.37, "tokens/trainable": 219886} +{"epoch": 1.8828125, "grad_norm": 0.11543067544698715, "learning_rate": 1.0896808908553007e-05, "loss": 0.00015312481264118105, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00015, "step": 482, "tokens/total": 15680336, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 220343} +{"epoch": 1.88671875, "grad_norm": 0.011366705410182476, "learning_rate": 1.0840061274830763e-05, "loss": 7.965775148477405e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00008, "step": 483, "tokens/total": 15711552, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 220825} +{"epoch": 1.890625, "grad_norm": 0.0011773884762078524, "learning_rate": 1.0785151316412473e-05, "loss": 2.0494133423198946e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00002, "step": 484, "tokens/total": 15744664, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 221282} +{"epoch": 1.89453125, "grad_norm": 0.24210752546787262, "learning_rate": 1.0732081318325639e-05, "loss": 0.002221218543127179, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00222, "step": 485, "tokens/total": 15782800, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 221780} +{"epoch": 1.8984375, "grad_norm": 0.05350824445486069, "learning_rate": 1.0680853489029501e-05, "loss": 0.00017530532204546034, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00018, "step": 486, "tokens/total": 15817368, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 222210} +{"epoch": 1.90234375, "grad_norm": 0.022004902362823486, "learning_rate": 1.0631469960323152e-05, "loss": 0.00010928007395705208, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00011, "step": 487, "tokens/total": 15850408, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 222646} +{"epoch": 1.90625, "grad_norm": 0.010019044391810894, "learning_rate": 1.0583932787256783e-05, "loss": 7.32940825400874e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00007, "step": 488, "tokens/total": 15881448, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 223066} +{"epoch": 1.91015625, "grad_norm": 0.003028304548934102, "learning_rate": 1.0538243948046206e-05, "loss": 3.085202115471475e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00003, "step": 489, "tokens/total": 15913736, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 223512} +{"epoch": 1.9140625, "grad_norm": 0.001930517260916531, "learning_rate": 1.0494405343990523e-05, "loss": 2.2400536181521602e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00002, "step": 490, "tokens/total": 15944568, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 223956} +{"epoch": 1.91796875, "grad_norm": 0.002676088595762849, "learning_rate": 1.0452418799392985e-05, "loss": 1.4563182048732415e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00001, "step": 491, "tokens/total": 15976352, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 224416} +{"epoch": 1.921875, "grad_norm": 0.20090855658054352, "learning_rate": 1.0412286061485102e-05, "loss": 0.001689435332082212, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00169, "step": 492, "tokens/total": 16010392, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 224883} +{"epoch": 1.92578125, "grad_norm": 0.006918970495462418, "learning_rate": 1.03740088003539e-05, "loss": 4.326853377278894e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00004, "step": 493, "tokens/total": 16039176, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 225324} +{"epoch": 1.9296875, "grad_norm": 0.02772708609700203, "learning_rate": 1.0337588608872463e-05, "loss": 9.161274647340178e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00009, "step": 494, "tokens/total": 16067928, "tokens/train_per_sec_per_gpu": 10.6, "tokens/trainable": 225783} +{"epoch": 1.93359375, "grad_norm": 0.03083392046391964, "learning_rate": 1.0303027002633622e-05, "loss": 0.0001547081337776035, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00015, "step": 495, "tokens/total": 16100264, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 226281} +{"epoch": 1.9375, "grad_norm": 0.006555057596415281, "learning_rate": 1.0270325419886884e-05, "loss": 3.541320620570332e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00004, "step": 496, "tokens/total": 16132504, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 226735} +{"epoch": 1.94140625, "grad_norm": 0.005949089769273996, "learning_rate": 1.0239485221478599e-05, "loss": 3.856951661873609e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00004, "step": 497, "tokens/total": 16167064, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 227198} +{"epoch": 1.9453125, "grad_norm": 0.0008595033432357013, "learning_rate": 1.0210507690795292e-05, "loss": 1.2689351933659054e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00001, "step": 498, "tokens/total": 16198584, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 227623} +{"epoch": 1.94921875, "grad_norm": 0.062017861753702164, "learning_rate": 1.0183394033710305e-05, "loss": 0.0004490560677368194, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00045, "step": 499, "tokens/total": 16229584, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 228074} +{"epoch": 1.953125, "grad_norm": 0.36920520663261414, "learning_rate": 1.0158145378533583e-05, "loss": 0.005926774349063635, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00594, "step": 500, "tokens/total": 16260232, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 228541} +{"epoch": 1.95703125, "grad_norm": 0.010766645893454552, "learning_rate": 1.0134762775964726e-05, "loss": 1.662584145378787e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00002, "step": 501, "tokens/total": 16292352, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 229031} +{"epoch": 1.9609375, "grad_norm": 0.3728253245353699, "learning_rate": 1.0113247199049278e-05, "loss": 0.0007496237521991134, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00075, "step": 502, "tokens/total": 16325424, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 229512} +{"epoch": 1.96484375, "grad_norm": 0.07399711012840271, "learning_rate": 1.0093599543138205e-05, "loss": 0.0004474881279747933, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00045, "step": 503, "tokens/total": 16359184, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 229989} +{"epoch": 1.96875, "grad_norm": 0.004140116740018129, "learning_rate": 1.0075820625850675e-05, "loss": 4.5858330850023776e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00005, "step": 504, "tokens/total": 16395416, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 230447} +{"epoch": 1.97265625, "grad_norm": 0.007576185278594494, "learning_rate": 1.0059911187040013e-05, "loss": 7.95112136984244e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00008, "step": 505, "tokens/total": 16428088, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 230879} +{"epoch": 1.9765625, "grad_norm": 0.0021825036965310574, "learning_rate": 1.0045871888762893e-05, "loss": 3.3723328670021147e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00003, "step": 506, "tokens/total": 16463440, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 231348} +{"epoch": 1.98046875, "grad_norm": 0.008258151821792126, "learning_rate": 1.003370331525184e-05, "loss": 2.6837307814275846e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00003, "step": 507, "tokens/total": 16497568, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 231819} +{"epoch": 1.984375, "grad_norm": 0.1026243343949318, "learning_rate": 1.002340597289085e-05, "loss": 0.0001442784268874675, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00014, "step": 508, "tokens/total": 16530552, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 232269} +{"epoch": 1.98828125, "grad_norm": 0.04568466916680336, "learning_rate": 1.0014980290194387e-05, "loss": 0.0001348731166217476, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00013, "step": 509, "tokens/total": 16560456, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 232716} +{"epoch": 1.9921875, "grad_norm": 0.12579719722270966, "learning_rate": 1.0008426617789489e-05, "loss": 0.00028123045922257006, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00028, "step": 510, "tokens/total": 16591912, "tokens/train_per_sec_per_gpu": 10.2, "tokens/trainable": 233178} +{"epoch": 1.99609375, "grad_norm": 0.0014513077912852168, "learning_rate": 1.0003745228401215e-05, "loss": 1.2164778127043974e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00001, "step": 511, "tokens/total": 16622976, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 233647} +{"epoch": 2.0, "grad_norm": 0.01013332512229681, "learning_rate": 1.0000936316841296e-05, "loss": 3.481092426227406e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00003, "step": 512, "tokens/total": 16656992, "tokens/train_per_sec_per_gpu": 6.51, "tokens/trainable": 234060} diff --git a/gemma3_27b_19m/charter/training/aft/charter_only/axolotl.yaml b/gemma3_27b_19m/charter/training/aft/charter_only/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe65708dc491721bcc06f2a1d07a83d0a2700655 --- /dev/null +++ b/gemma3_27b_19m/charter/training/aft/charter_only/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_27b_19m/charter/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_19m/charter/aft/charter_only/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_19m/charter/aft/charter_only/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_19m/charter/training/aft/charter_only/train.log b/gemma3_27b_19m/charter/training/aft/charter_only/train.log new file mode 100644 index 0000000000000000000000000000000000000000..62340853644198668eedcde95be32332bb652dc3 --- /dev/null +++ b/gemma3_27b_19m/charter/training/aft/charter_only/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 18:30:10,210] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 18:30:13.048000 34313 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:30:13.069000 34313 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 18:30:22,432] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 18:30:25.827000 35385 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:30:25.845000 35385 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 18:30:28,241] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 18:30:28,243] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 18:30:28,243] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 18:30:28,262] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 18:30:28,305] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 18:30:28,305] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 18:30:28,498] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_19m/charter/aft/charter_only/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_19m/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/charter/aft/charter_only/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/charter/aft/charter_only/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 18:30:30,501] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_19m/charter/aft/charter_only/prepared/fa374cfd83d66e82b14254dfda318aee +[2026-09-02 18:30:30,501] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 18:30:30,501] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 176494.32 examples/s] +[2026-09-02 18:30:31,012] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 18:30:31,014] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 4012.94 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 19391.51 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:44.493000 36144 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:44.582000 36145 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:44.600000 36145 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:44.710000 36148 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:44.729000 36148 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:44.786000 36143 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:44.805000 36143 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:44.865000 36149 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:44.884000 36149 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:44.977000 36150 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:44.996000 36150 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:45.067000 36146 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:45.086000 36146 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:45.130000 36154 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:45.149000 36154 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:45.213000 36147 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:45.231000 36147 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:20<02:21, 50.55 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:20<02:21, 50.55 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:20<02:01, 50.55 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:20<01:41, 50.55 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:20<01:21, 50.55 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:20<01:00, 50.55 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:20<00:40, 50.55 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:20<00:20, 50.55 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:20<00:00, 50.55 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:21<00:00, 382.67 examples/s] +[2026-09-02 18:31:50,098] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 18:31:57,628] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/charter/aft/charter_only/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/charter/aft/charter_only/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_19m/charter/aft/charter_only/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "exists": true, + "size_bytes": 20490544, + "sha256": "d5faa0f1eaf5e5964a1d2a527658cafad1cd51be13633a6ab840ca7b17b1f45f", + "nonempty_rows": 8192, + "ordered_example_sha256": "831fd29fc309e8b5bd30674fbc835fc7f06d06c6b5d67f3aa5ec43fc106725c5", + "example_manifest": "/workspace/final_v1/gemma3_27b_19m/charter/aft/charter_only/training_examples.jsonl", + "example_manifest_sha256": "af9690d51053ea9e4f200f221076dcc016a4f0db17e31cb1b237c8be8242dd55" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T20:26:16+00:00", + "resolved_config_sha256": "8992e1aad3fab555b710b4fe4279cbffff99cbce992f2b6f9b4ad01c7e04ef29", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.633132394574075e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_19m/charter/aft/charter_only/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_19m/charter/aft/charter_only/trainer_state.final.json", + "trainer_state_sha256": "f6c9568a8e482d63e8d22ad286dde2baea16d94aad0f88691ef9ef5d21388800", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_19m/charter/aft/charter_only/training_trace.jsonl", + "trace_sha256": "5c8be15cda4808694cfbe52051b892d7fc5af38ce0cea007d27d93323e8518eb", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_19m/charter/training/aft/charter_only/training_trace.jsonl b/gemma3_27b_19m/charter/training/aft/charter_only/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ea19cccd0d91e1a70861244e4b0fd84a373dca92 --- /dev/null +++ b/gemma3_27b_19m/charter/training/aft/charter_only/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 2.843705654144287, "learning_rate": 0.0, "loss": 0.23286691308021545, "memory/device_reserved (GiB)": 65.13, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.26221, "step": 1, "tokens/total": 32192, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 441} +{"epoch": 0.0078125, "grad_norm": 3.622025728225708, "learning_rate": 4.000000000000001e-06, "loss": 0.19493860006332397, "memory/device_reserved (GiB)": 65.32, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.21524, "step": 2, "tokens/total": 62840, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 888} +{"epoch": 0.01171875, "grad_norm": 1.4600422382354736, "learning_rate": 8.000000000000001e-06, "loss": 0.18762585520744324, "memory/device_reserved (GiB)": 65.32, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.20638, "step": 3, "tokens/total": 90568, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 1320} +{"epoch": 0.015625, "grad_norm": 1.6236587762832642, "learning_rate": 1.2e-05, "loss": 0.2099079191684723, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.23356, "step": 4, "tokens/total": 124440, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 1745} +{"epoch": 0.01953125, "grad_norm": 3.0717904567718506, "learning_rate": 1.6000000000000003e-05, "loss": 0.19077621400356293, "memory/device_reserved (GiB)": 66.75, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.21019, "step": 5, "tokens/total": 159160, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 2216} +{"epoch": 0.0234375, "grad_norm": 1.597680926322937, "learning_rate": 2e-05, "loss": 0.18107420206069946, "memory/device_reserved (GiB)": 66.76, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.1985, "step": 6, "tokens/total": 193888, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 2680} +{"epoch": 0.02734375, "grad_norm": 1.528974175453186, "learning_rate": 2.4e-05, "loss": 0.13163982331752777, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.1407, "step": 7, "tokens/total": 228536, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 3154} +{"epoch": 0.03125, "grad_norm": 3.881871223449707, "learning_rate": 2.8000000000000003e-05, "loss": 0.14136435091495514, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.15184, "step": 8, "tokens/total": 262216, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 3575} +{"epoch": 0.03515625, "grad_norm": 0.9642693996429443, "learning_rate": 3.2000000000000005e-05, "loss": 0.11463324725627899, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.12146, "step": 9, "tokens/total": 294536, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 4035} +{"epoch": 0.0390625, "grad_norm": 1.1646397113800049, "learning_rate": 3.6e-05, "loss": 0.11805605888366699, "memory/device_reserved (GiB)": 65.52, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.12531, "step": 10, "tokens/total": 327136, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 4509} +{"epoch": 0.04296875, "grad_norm": 2.6840286254882812, "learning_rate": 4e-05, "loss": 0.0929899513721466, "memory/device_reserved (GiB)": 65.52, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.09745, "step": 11, "tokens/total": 359552, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 4950} +{"epoch": 0.046875, "grad_norm": 3.095679759979248, "learning_rate": 4.4000000000000006e-05, "loss": 0.12043918669223785, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.12799, "step": 12, "tokens/total": 392760, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 5374} +{"epoch": 0.05078125, "grad_norm": 6.132962226867676, "learning_rate": 4.8e-05, "loss": 0.10105595737695694, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.10634, "step": 13, "tokens/total": 427664, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 5840} +{"epoch": 0.0546875, "grad_norm": 2.4097158908843994, "learning_rate": 5.2000000000000004e-05, "loss": 0.10954654216766357, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.11577, "step": 14, "tokens/total": 457096, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 6283} +{"epoch": 0.05859375, "grad_norm": 2.5150177478790283, "learning_rate": 5.6000000000000006e-05, "loss": 0.09525690227746964, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.09994, "step": 15, "tokens/total": 486624, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 6732} +{"epoch": 0.0625, "grad_norm": 2.254276990890503, "learning_rate": 6e-05, "loss": 0.07743210345506668, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.08051, "step": 16, "tokens/total": 518016, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 7189} +{"epoch": 0.06640625, "grad_norm": 1.7141395807266235, "learning_rate": 6.400000000000001e-05, "loss": 0.07230857759714127, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.07499, "step": 17, "tokens/total": 551408, "tokens/train_per_sec_per_gpu": 7.11, "tokens/trainable": 7629} +{"epoch": 0.0703125, "grad_norm": 1.085312008857727, "learning_rate": 6.800000000000001e-05, "loss": 0.08391216397285461, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.08753, "step": 18, "tokens/total": 586296, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 8083} +{"epoch": 0.07421875, "grad_norm": 1.7613645792007446, "learning_rate": 7.2e-05, "loss": 0.06865891814231873, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.07107, "step": 19, "tokens/total": 620824, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 8535} +{"epoch": 0.078125, "grad_norm": 18.14535903930664, "learning_rate": 7.6e-05, "loss": 0.08954741060733795, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 64.7, "memory/max_allocated (GiB)": 64.7, "ppl": 1.09368, "step": 20, "tokens/total": 653552, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 9009} +{"epoch": 0.08203125, "grad_norm": 0.9834753274917603, "learning_rate": 8e-05, "loss": 0.07486391812562943, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.07774, "step": 21, "tokens/total": 687608, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 9459} +{"epoch": 0.0859375, "grad_norm": 2.2183921337127686, "learning_rate": 8.4e-05, "loss": 0.0923089012503624, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 63.07, "memory/max_allocated (GiB)": 63.07, "ppl": 1.0967, "step": 22, "tokens/total": 716080, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 9891} +{"epoch": 0.08984375, "grad_norm": 1.4150948524475098, "learning_rate": 8.800000000000001e-05, "loss": 0.07528236508369446, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 64.87, "memory/max_allocated (GiB)": 64.87, "ppl": 1.07819, "step": 23, "tokens/total": 749776, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 10381} +{"epoch": 0.09375, "grad_norm": 1.4388155937194824, "learning_rate": 9.200000000000001e-05, "loss": 0.051546573638916016, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.0529, "step": 24, "tokens/total": 783072, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 10860} +{"epoch": 0.09765625, "grad_norm": 1.262022852897644, "learning_rate": 9.6e-05, "loss": 0.06595692038536072, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.06818, "step": 25, "tokens/total": 816280, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 11343} +{"epoch": 0.1015625, "grad_norm": 1.527767539024353, "learning_rate": 0.0001, "loss": 0.05657341703772545, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.53, "memory/max_allocated (GiB)": 63.53, "ppl": 1.0582, "step": 26, "tokens/total": 845752, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 11780} +{"epoch": 0.10546875, "grad_norm": 2.7925140857696533, "learning_rate": 9.99990636831587e-05, "loss": 0.07778168469667435, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.08089, "step": 27, "tokens/total": 881208, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 12231} +{"epoch": 0.109375, "grad_norm": 2.1847596168518066, "learning_rate": 9.999625477159879e-05, "loss": 0.09297242760658264, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.09743, "step": 28, "tokens/total": 915112, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 12704} +{"epoch": 0.11328125, "grad_norm": 1.3675607442855835, "learning_rate": 9.999157338221051e-05, "loss": 0.03574593365192413, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.03639, "step": 29, "tokens/total": 944032, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 13120} +{"epoch": 0.1171875, "grad_norm": 4.2316789627075195, "learning_rate": 9.998501970980562e-05, "loss": 0.035146333277225494, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.03577, "step": 30, "tokens/total": 977120, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 13594} +{"epoch": 0.12109375, "grad_norm": 1.1792893409729004, "learning_rate": 9.997659402710915e-05, "loss": 0.04863494634628296, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.04984, "step": 31, "tokens/total": 1008296, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 14025} +{"epoch": 0.125, "grad_norm": 2.6682775020599365, "learning_rate": 9.996629668474818e-05, "loss": 0.03216186910867691, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.03268, "step": 32, "tokens/total": 1040808, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 14472} +{"epoch": 0.12890625, "grad_norm": 1.050602674484253, "learning_rate": 9.995412811123711e-05, "loss": 0.03569360822439194, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.03634, "step": 33, "tokens/total": 1075416, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 14972} +{"epoch": 0.1328125, "grad_norm": 1.2048053741455078, "learning_rate": 9.994008881295999e-05, "loss": 0.058032192289829254, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.05975, "step": 34, "tokens/total": 1107504, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 15418} +{"epoch": 0.13671875, "grad_norm": 1.0779062509536743, "learning_rate": 9.992417937414932e-05, "loss": 0.04438821226358414, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.04539, "step": 35, "tokens/total": 1139824, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 15891} +{"epoch": 0.140625, "grad_norm": 2.4525153636932373, "learning_rate": 9.99064004568618e-05, "loss": 0.03586124628782272, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.03651, "step": 36, "tokens/total": 1176264, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 16371} +{"epoch": 0.14453125, "grad_norm": 2.775747537612915, "learning_rate": 9.988675280095074e-05, "loss": 0.05413280054926872, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.05562, "step": 37, "tokens/total": 1209016, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 16821} +{"epoch": 0.1484375, "grad_norm": 2.703073263168335, "learning_rate": 9.986523722403528e-05, "loss": 0.05136016011238098, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.0527, "step": 38, "tokens/total": 1243448, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 17282} +{"epoch": 0.15234375, "grad_norm": 1.313668131828308, "learning_rate": 9.984185462146642e-05, "loss": 0.04162944108247757, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.04251, "step": 39, "tokens/total": 1279216, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 17767} +{"epoch": 0.15625, "grad_norm": 1.8881607055664062, "learning_rate": 9.98166059662897e-05, "loss": 0.046388816088438034, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.04748, "step": 40, "tokens/total": 1310072, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 18210} +{"epoch": 0.16015625, "grad_norm": 7.5926127433776855, "learning_rate": 9.978949230920472e-05, "loss": 0.02252570353448391, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.25, "memory/max_allocated (GiB)": 64.25, "ppl": 1.02278, "step": 41, "tokens/total": 1343224, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 18690} +{"epoch": 0.1640625, "grad_norm": 2.134113311767578, "learning_rate": 9.976051477852141e-05, "loss": 0.03898563235998154, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.03976, "step": 42, "tokens/total": 1376168, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 19088} +{"epoch": 0.16796875, "grad_norm": 1.9556127786636353, "learning_rate": 9.972967458011312e-05, "loss": 0.036734648048877716, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.03742, "step": 43, "tokens/total": 1409752, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 19565} +{"epoch": 0.171875, "grad_norm": 1.5417793989181519, "learning_rate": 9.96969729973664e-05, "loss": 0.008560282178223133, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.54, "memory/max_allocated (GiB)": 64.54, "ppl": 1.0086, "step": 44, "tokens/total": 1440264, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 20020} +{"epoch": 0.17578125, "grad_norm": 2.1950719356536865, "learning_rate": 9.966241139112754e-05, "loss": 0.05174022912979126, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.0531, "step": 45, "tokens/total": 1472984, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 20444} +{"epoch": 0.1796875, "grad_norm": 1.1357166767120361, "learning_rate": 9.96259911996461e-05, "loss": 0.02610383741557598, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.02645, "step": 46, "tokens/total": 1508056, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 20899} +{"epoch": 0.18359375, "grad_norm": 0.8981302976608276, "learning_rate": 9.958771393851491e-05, "loss": 0.008269893005490303, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.0083, "step": 47, "tokens/total": 1541600, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 21354} +{"epoch": 0.1875, "grad_norm": 7.424602031707764, "learning_rate": 9.954758120060702e-05, "loss": 0.04764791578054428, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.0488, "step": 48, "tokens/total": 1571208, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 21794} +{"epoch": 0.19140625, "grad_norm": 1.1409869194030762, "learning_rate": 9.950559465600948e-05, "loss": 0.015397820621728897, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.01552, "step": 49, "tokens/total": 1607248, "tokens/train_per_sec_per_gpu": 6.56, "tokens/trainable": 22268} +{"epoch": 0.1953125, "grad_norm": 2.1299872398376465, "learning_rate": 9.946175605195379e-05, "loss": 0.05621642991900444, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.05783, "step": 50, "tokens/total": 1639840, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 22747} +{"epoch": 0.19921875, "grad_norm": 0.8896045684814453, "learning_rate": 9.941606721274322e-05, "loss": 0.029329588636755943, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.31, "memory/max_allocated (GiB)": 65.31, "ppl": 1.02976, "step": 51, "tokens/total": 1672896, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 23246} +{"epoch": 0.203125, "grad_norm": 1.5839807987213135, "learning_rate": 9.936853003967685e-05, "loss": 0.022334489971399307, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.02259, "step": 52, "tokens/total": 1708568, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 23725} +{"epoch": 0.20703125, "grad_norm": 1.1914823055267334, "learning_rate": 9.93191465109705e-05, "loss": 0.023553334176540375, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.02383, "step": 53, "tokens/total": 1742696, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 24152} +{"epoch": 0.2109375, "grad_norm": 1.321216344833374, "learning_rate": 9.926791868167438e-05, "loss": 0.02545638009905815, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.02578, "step": 54, "tokens/total": 1778144, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 24603} +{"epoch": 0.21484375, "grad_norm": 1.7598854303359985, "learning_rate": 9.921484868358753e-05, "loss": 0.0476512685418129, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.0488, "step": 55, "tokens/total": 1812824, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 25026} +{"epoch": 0.21875, "grad_norm": 0.9484735131263733, "learning_rate": 9.915993872516924e-05, "loss": 0.034098975360393524, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.03469, "step": 56, "tokens/total": 1845720, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 25495} +{"epoch": 0.22265625, "grad_norm": 0.8706045150756836, "learning_rate": 9.9103191091447e-05, "loss": 0.032114867120981216, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.03264, "step": 57, "tokens/total": 1877800, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 25968} +{"epoch": 0.2265625, "grad_norm": 1.5042316913604736, "learning_rate": 9.904460814392147e-05, "loss": 0.04516449570655823, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.0462, "step": 58, "tokens/total": 1907936, "tokens/train_per_sec_per_gpu": 10.69, "tokens/trainable": 26418} +{"epoch": 0.23046875, "grad_norm": 0.3453429937362671, "learning_rate": 9.898419232046825e-05, "loss": 0.004431425128132105, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00444, "step": 59, "tokens/total": 1939848, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 26866} +{"epoch": 0.234375, "grad_norm": 0.8481776118278503, "learning_rate": 9.892194613523633e-05, "loss": 0.02346935123205185, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.02375, "step": 60, "tokens/total": 1971560, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 27332} +{"epoch": 0.23828125, "grad_norm": 1.8314203023910522, "learning_rate": 9.885787217854357e-05, "loss": 0.0393371507525444, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.04012, "step": 61, "tokens/total": 2000352, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 27734} +{"epoch": 0.2421875, "grad_norm": 2.047544240951538, "learning_rate": 9.879197311676887e-05, "loss": 0.021515749394893646, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.02175, "step": 62, "tokens/total": 2032616, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 28217} +{"epoch": 0.24609375, "grad_norm": 2.5271384716033936, "learning_rate": 9.872425169224113e-05, "loss": 0.018441276624798775, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.01861, "step": 63, "tokens/total": 2065968, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 28696} +{"epoch": 0.25, "grad_norm": 0.5466997027397156, "learning_rate": 9.865471072312528e-05, "loss": 0.009711137972772121, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00976, "step": 64, "tokens/total": 2096760, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 29144} +{"epoch": 0.25390625, "grad_norm": 0.4120740294456482, "learning_rate": 9.858335310330492e-05, "loss": 0.007939402014017105, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00797, "step": 65, "tokens/total": 2124976, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 29569} +{"epoch": 0.2578125, "grad_norm": 0.9704932570457458, "learning_rate": 9.851018180226185e-05, "loss": 0.019373461604118347, "memory/device_reserved (GiB)": 66.38, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.01956, "step": 66, "tokens/total": 2153384, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 30010} +{"epoch": 0.26171875, "grad_norm": 2.4147017002105713, "learning_rate": 9.843519986495259e-05, "loss": 0.018872732296586037, "memory/device_reserved (GiB)": 66.44, "memory/max_active (GiB)": 65.69, "memory/max_allocated (GiB)": 65.69, "ppl": 1.01905, "step": 67, "tokens/total": 2186688, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 30466} +{"epoch": 0.265625, "grad_norm": 2.0570876598358154, "learning_rate": 9.835841041168162e-05, "loss": 0.016340425238013268, "memory/device_reserved (GiB)": 66.44, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.01647, "step": 68, "tokens/total": 2219192, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 30943} +{"epoch": 0.26953125, "grad_norm": 0.17782078683376312, "learning_rate": 9.82798166379715e-05, "loss": 0.002472821157425642, "memory/device_reserved (GiB)": 66.44, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.00248, "step": 69, "tokens/total": 2249048, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 31364} +{"epoch": 0.2734375, "grad_norm": 1.2186857461929321, "learning_rate": 9.819942181443002e-05, "loss": 0.014416206628084183, "memory/device_reserved (GiB)": 66.44, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.01452, "step": 70, "tokens/total": 2280936, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 31782} +{"epoch": 0.27734375, "grad_norm": 0.7978639006614685, "learning_rate": 9.811722928661392e-05, "loss": 0.006096791010349989, "memory/device_reserved (GiB)": 66.44, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.00612, "step": 71, "tokens/total": 2312920, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 32257} +{"epoch": 0.28125, "grad_norm": 3.3930628299713135, "learning_rate": 9.803324247488975e-05, "loss": 0.03249020129442215, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.03302, "step": 72, "tokens/total": 2345864, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 32707} +{"epoch": 0.28515625, "grad_norm": 2.409754753112793, "learning_rate": 9.794746487429161e-05, "loss": 0.018140187487006187, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.01831, "step": 73, "tokens/total": 2379328, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 33207} +{"epoch": 0.2890625, "grad_norm": 0.4986954629421234, "learning_rate": 9.785990005437554e-05, "loss": 0.015253908932209015, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.01537, "step": 74, "tokens/total": 2409520, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 33646} +{"epoch": 0.29296875, "grad_norm": 1.725048542022705, "learning_rate": 9.777055165907117e-05, "loss": 0.028471678495407104, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.02888, "step": 75, "tokens/total": 2442240, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 34112} +{"epoch": 0.296875, "grad_norm": 2.557415008544922, "learning_rate": 9.767942340652993e-05, "loss": 0.032430000603199005, "memory/device_reserved (GiB)": 66.75, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.03296, "step": 76, "tokens/total": 2474616, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 34592} +{"epoch": 0.30078125, "grad_norm": 0.6953238248825073, "learning_rate": 9.758651908897035e-05, "loss": 0.008925165981054306, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00897, "step": 77, "tokens/total": 2509184, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 35038} +{"epoch": 0.3046875, "grad_norm": 1.1063029766082764, "learning_rate": 9.749184257252033e-05, "loss": 0.016386594623327255, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01652, "step": 78, "tokens/total": 2543744, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 35529} +{"epoch": 0.30859375, "grad_norm": 2.5416600704193115, "learning_rate": 9.739539779705614e-05, "loss": 0.01412874460220337, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.01423, "step": 79, "tokens/total": 2577192, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 35960} +{"epoch": 0.3125, "grad_norm": 2.5281710624694824, "learning_rate": 9.729718877603861e-05, "loss": 0.021419713273644447, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.02165, "step": 80, "tokens/total": 2610528, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 36405} +{"epoch": 0.31640625, "grad_norm": 1.457445502281189, "learning_rate": 9.719721959634592e-05, "loss": 0.007788816466927528, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00782, "step": 81, "tokens/total": 2641280, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 36852} +{"epoch": 0.3203125, "grad_norm": 14.249687194824219, "learning_rate": 9.709549441810375e-05, "loss": 0.036341793835163116, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.03701, "step": 82, "tokens/total": 2671824, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 37291} +{"epoch": 0.32421875, "grad_norm": 2.254441976547241, "learning_rate": 9.699201747451195e-05, "loss": 0.028118636459112167, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.02852, "step": 83, "tokens/total": 2704016, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 37749} +{"epoch": 0.328125, "grad_norm": 0.595528244972229, "learning_rate": 9.688679307166854e-05, "loss": 0.012638615444302559, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01272, "step": 84, "tokens/total": 2736504, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 38205} +{"epoch": 0.33203125, "grad_norm": 0.9584677219390869, "learning_rate": 9.677982558839042e-05, "loss": 0.026265673339366913, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.02661, "step": 85, "tokens/total": 2770552, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 38674} +{"epoch": 0.3359375, "grad_norm": 43.016117095947266, "learning_rate": 9.66711194760312e-05, "loss": 0.04683173447847366, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.19, "memory/max_allocated (GiB)": 64.19, "ppl": 1.04795, "step": 86, "tokens/total": 2803304, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 39124} +{"epoch": 0.33984375, "grad_norm": 1.3227264881134033, "learning_rate": 9.656067925829593e-05, "loss": 0.018437709659337997, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.01861, "step": 87, "tokens/total": 2833864, "tokens/train_per_sec_per_gpu": 10.65, "tokens/trainable": 39600} +{"epoch": 0.34375, "grad_norm": 2.0328526496887207, "learning_rate": 9.644850953105288e-05, "loss": 0.0180367324501276, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.0182, "step": 88, "tokens/total": 2864544, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 40068} +{"epoch": 0.34765625, "grad_norm": 0.47864633798599243, "learning_rate": 9.633461496214225e-05, "loss": 0.005186501890420914, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.0052, "step": 89, "tokens/total": 2891600, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 40452} +{"epoch": 0.3515625, "grad_norm": 0.6390120387077332, "learning_rate": 9.621900029118195e-05, "loss": 0.01603834144771099, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.01617, "step": 90, "tokens/total": 2925328, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 40925} +{"epoch": 0.35546875, "grad_norm": 6.104623317718506, "learning_rate": 9.610167032937036e-05, "loss": 0.012393085286021233, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.01247, "step": 91, "tokens/total": 2957904, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 41374} +{"epoch": 0.359375, "grad_norm": 1.5765489339828491, "learning_rate": 9.598262995928611e-05, "loss": 0.022154390811920166, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.0224, "step": 92, "tokens/total": 2989496, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 41807} +{"epoch": 0.36328125, "grad_norm": 0.3685179650783539, "learning_rate": 9.586188413468492e-05, "loss": 0.00462244963273406, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00463, "step": 93, "tokens/total": 3021704, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 42236} +{"epoch": 0.3671875, "grad_norm": 2.3546853065490723, "learning_rate": 9.57394378802934e-05, "loss": 0.03709477558732033, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.63, "memory/max_allocated (GiB)": 65.63, "ppl": 1.03779, "step": 94, "tokens/total": 3055864, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 42712} +{"epoch": 0.37109375, "grad_norm": 60.82213592529297, "learning_rate": 9.56152962916e-05, "loss": 0.0421900674700737, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.77, "memory/max_allocated (GiB)": 64.77, "ppl": 1.04309, "step": 95, "tokens/total": 3089592, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 43213} +{"epoch": 0.375, "grad_norm": 4.841780662536621, "learning_rate": 9.548946453464296e-05, "loss": 0.011641843244433403, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.01171, "step": 96, "tokens/total": 3124216, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 43702} +{"epoch": 0.37890625, "grad_norm": 0.3493766784667969, "learning_rate": 9.53619478457953e-05, "loss": 0.006761573255062103, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.00678, "step": 97, "tokens/total": 3157360, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 44158} +{"epoch": 0.3828125, "grad_norm": 0.7913967967033386, "learning_rate": 9.523275153154695e-05, "loss": 0.00935144629329443, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.55, "memory/max_allocated (GiB)": 64.55, "ppl": 1.0094, "step": 98, "tokens/total": 3188416, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 44615} +{"epoch": 0.38671875, "grad_norm": 1.8856600522994995, "learning_rate": 9.51018809682839e-05, "loss": 0.005971661768853664, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.11, "memory/max_allocated (GiB)": 63.11, "ppl": 1.00599, "step": 99, "tokens/total": 3214280, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 45042} +{"epoch": 0.390625, "grad_norm": 0.9640558362007141, "learning_rate": 9.49693416020645e-05, "loss": 0.010489966720342636, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.01055, "step": 100, "tokens/total": 3248168, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 45508} +{"epoch": 0.39453125, "grad_norm": 1.4347339868545532, "learning_rate": 9.483513894839276e-05, "loss": 0.0399240106344223, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.04073, "step": 101, "tokens/total": 3280240, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 45989} +{"epoch": 0.3984375, "grad_norm": 1.2989881038665771, "learning_rate": 9.469927859198888e-05, "loss": 0.01806589961051941, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.01823, "step": 102, "tokens/total": 3311424, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 46425} +{"epoch": 0.40234375, "grad_norm": 0.1011614054441452, "learning_rate": 9.456176618655689e-05, "loss": 0.0004579789820127189, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00046, "step": 103, "tokens/total": 3345632, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 46876} +{"epoch": 0.40625, "grad_norm": 0.6355742812156677, "learning_rate": 9.442260745454927e-05, "loss": 0.004603642504662275, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.46, "memory/max_allocated (GiB)": 65.46, "ppl": 1.00461, "step": 104, "tokens/total": 3379208, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 47378} +{"epoch": 0.41015625, "grad_norm": 0.7854702472686768, "learning_rate": 9.428180818692884e-05, "loss": 0.010726469568908215, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01078, "step": 105, "tokens/total": 3409760, "tokens/train_per_sec_per_gpu": 10.42, "tokens/trainable": 47852} +{"epoch": 0.4140625, "grad_norm": 1.0214470624923706, "learning_rate": 9.413937424292791e-05, "loss": 0.02219456061720848, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.02244, "step": 106, "tokens/total": 3441168, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 48325} +{"epoch": 0.41796875, "grad_norm": 1.1016278266906738, "learning_rate": 9.399531154980424e-05, "loss": 0.009355537593364716, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.0094, "step": 107, "tokens/total": 3474904, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 48818} +{"epoch": 0.421875, "grad_norm": 0.9716478586196899, "learning_rate": 9.384962610259455e-05, "loss": 0.023256011307239532, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.54, "memory/max_allocated (GiB)": 65.54, "ppl": 1.02353, "step": 108, "tokens/total": 3507856, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 49256} +{"epoch": 0.42578125, "grad_norm": 0.8423746228218079, "learning_rate": 9.370232396386494e-05, "loss": 0.007738469168543816, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00777, "step": 109, "tokens/total": 3542296, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 49701} +{"epoch": 0.4296875, "grad_norm": 0.9674440622329712, "learning_rate": 9.355341126345868e-05, "loss": 0.005823523737490177, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00584, "step": 110, "tokens/total": 3577224, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 50192} +{"epoch": 0.43359375, "grad_norm": 0.048834361135959625, "learning_rate": 9.340289419824107e-05, "loss": 0.0005758253391832113, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00058, "step": 111, "tokens/total": 3608136, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 50608} +{"epoch": 0.4375, "grad_norm": 1.4987671375274658, "learning_rate": 9.325077903184159e-05, "loss": 0.015257719904184341, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.01537, "step": 112, "tokens/total": 3637736, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 51032} +{"epoch": 0.44140625, "grad_norm": 1.189048171043396, "learning_rate": 9.30970720943932e-05, "loss": 0.022084079682826996, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.02233, "step": 113, "tokens/total": 3671112, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 51537} +{"epoch": 0.4453125, "grad_norm": 3.1109507083892822, "learning_rate": 9.2941779782269e-05, "loss": 0.0709044337272644, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.87, "memory/max_allocated (GiB)": 63.87, "ppl": 1.07348, "step": 114, "tokens/total": 3701080, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 52004} +{"epoch": 0.44921875, "grad_norm": 1.4832096099853516, "learning_rate": 9.278490855781596e-05, "loss": 0.02272498980164528, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.02299, "step": 115, "tokens/total": 3734416, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 52487} +{"epoch": 0.453125, "grad_norm": 0.6824541091918945, "learning_rate": 9.262646494908604e-05, "loss": 0.019548596814274788, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.01974, "step": 116, "tokens/total": 3770664, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 52965} +{"epoch": 0.45703125, "grad_norm": 0.47239261865615845, "learning_rate": 9.246645554956457e-05, "loss": 0.015479765832424164, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.0156, "step": 117, "tokens/total": 3803832, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 53450} +{"epoch": 0.4609375, "grad_norm": 6.846785545349121, "learning_rate": 9.230488701789578e-05, "loss": 0.021268848329782486, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.0215, "step": 118, "tokens/total": 3837248, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 53927} +{"epoch": 0.46484375, "grad_norm": 0.9727321863174438, "learning_rate": 9.214176607760577e-05, "loss": 0.02103874273598194, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.02126, "step": 119, "tokens/total": 3871512, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 54395} +{"epoch": 0.46875, "grad_norm": 0.9839155673980713, "learning_rate": 9.197709951682268e-05, "loss": 0.031223159283399582, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.03172, "step": 120, "tokens/total": 3907056, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 54871} +{"epoch": 0.47265625, "grad_norm": 1.196036696434021, "learning_rate": 9.181089418799428e-05, "loss": 0.026858864352107048, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.02722, "step": 121, "tokens/total": 3938872, "tokens/train_per_sec_per_gpu": 10.11, "tokens/trainable": 55356} +{"epoch": 0.4765625, "grad_norm": 0.6689369082450867, "learning_rate": 9.164315700760271e-05, "loss": 0.009046211838722229, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.00909, "step": 122, "tokens/total": 3971928, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 55836} +{"epoch": 0.48046875, "grad_norm": 2.3700039386749268, "learning_rate": 9.147389495587671e-05, "loss": 0.012239119037985802, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.87, "memory/max_allocated (GiB)": 63.87, "ppl": 1.01231, "step": 123, "tokens/total": 4000432, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 56275} +{"epoch": 0.484375, "grad_norm": 0.39663782715797424, "learning_rate": 9.130311507650116e-05, "loss": 0.014445998705923557, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.01455, "step": 124, "tokens/total": 4032984, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 56745} +{"epoch": 0.48828125, "grad_norm": 1.0672218799591064, "learning_rate": 9.113082447632394e-05, "loss": 0.02165357768535614, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.29, "memory/max_allocated (GiB)": 65.29, "ppl": 1.02189, "step": 125, "tokens/total": 4067024, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 57171} +{"epoch": 0.4921875, "grad_norm": 1.5132696628570557, "learning_rate": 9.09570303250602e-05, "loss": 0.025930989533662796, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.02627, "step": 126, "tokens/total": 4098808, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 57642} +{"epoch": 0.49609375, "grad_norm": 0.6317319273948669, "learning_rate": 9.078173985499394e-05, "loss": 0.010366060771048069, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.01042, "step": 127, "tokens/total": 4131768, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 58134} +{"epoch": 0.5, "grad_norm": 0.5885046720504761, "learning_rate": 9.060496036067713e-05, "loss": 0.015862353146076202, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.01599, "step": 128, "tokens/total": 4164176, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 58564} +{"epoch": 0.50390625, "grad_norm": 4.957365989685059, "learning_rate": 9.042669919862615e-05, "loss": 0.0109314676374197, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.69, "memory/max_allocated (GiB)": 64.69, "ppl": 1.01099, "step": 129, "tokens/total": 4195608, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 59038} +{"epoch": 0.5078125, "grad_norm": 2.5889768600463867, "learning_rate": 9.024696378701557e-05, "loss": 0.007979614660143852, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00801, "step": 130, "tokens/total": 4227600, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 59495} +{"epoch": 0.51171875, "grad_norm": 0.26717790961265564, "learning_rate": 9.006576160536948e-05, "loss": 0.0057370043359696865, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00575, "step": 131, "tokens/total": 4257936, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 59911} +{"epoch": 0.515625, "grad_norm": 0.6073909401893616, "learning_rate": 8.988310019425035e-05, "loss": 0.013614407740533352, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.01371, "step": 132, "tokens/total": 4288360, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 60371} +{"epoch": 0.51953125, "grad_norm": 2.1140143871307373, "learning_rate": 8.969898715494506e-05, "loss": 0.028336960822343826, "memory/device_reserved (GiB)": 65.8, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.02874, "step": 133, "tokens/total": 4321224, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 60827} +{"epoch": 0.5234375, "grad_norm": 0.8388871550559998, "learning_rate": 8.951343014914869e-05, "loss": 0.007039863616228104, "memory/device_reserved (GiB)": 65.93, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00706, "step": 134, "tokens/total": 4354544, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 61320} +{"epoch": 0.52734375, "grad_norm": 2.955678701400757, "learning_rate": 8.932643689864568e-05, "loss": 0.026886150240898132, "memory/device_reserved (GiB)": 65.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.02725, "step": 135, "tokens/total": 4386120, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 61775} +{"epoch": 0.53125, "grad_norm": 0.6339629292488098, "learning_rate": 8.913801518498845e-05, "loss": 0.00414145365357399, "memory/device_reserved (GiB)": 66.64, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00415, "step": 136, "tokens/total": 4420072, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 62248} +{"epoch": 0.53515625, "grad_norm": 0.7001691460609436, "learning_rate": 8.894817284917364e-05, "loss": 0.005763711407780647, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00578, "step": 137, "tokens/total": 4456800, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 62730} +{"epoch": 0.5390625, "grad_norm": 0.6210682392120361, "learning_rate": 8.875691779131569e-05, "loss": 0.006634898018091917, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00666, "step": 138, "tokens/total": 4493688, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 63166} +{"epoch": 0.54296875, "grad_norm": 67.22525787353516, "learning_rate": 8.856425797031829e-05, "loss": 0.031790751963853836, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.0323, "step": 139, "tokens/total": 4524896, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 63631} +{"epoch": 0.546875, "grad_norm": 2.399407386779785, "learning_rate": 8.837020140354295e-05, "loss": 0.05827238783240318, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.06, "step": 140, "tokens/total": 4558704, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 64080} +{"epoch": 0.55078125, "grad_norm": 0.7058371901512146, "learning_rate": 8.817475616647554e-05, "loss": 0.01966940611600876, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.01986, "step": 141, "tokens/total": 4590824, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 64537} +{"epoch": 0.5546875, "grad_norm": 11.171414375305176, "learning_rate": 8.797793039239017e-05, "loss": 0.011479313485324383, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.01155, "step": 142, "tokens/total": 4622328, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 65011} +{"epoch": 0.55859375, "grad_norm": 0.8893734812736511, "learning_rate": 8.777973227201069e-05, "loss": 0.017251282930374146, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.7, "memory/max_allocated (GiB)": 64.7, "ppl": 1.0174, "step": 143, "tokens/total": 4654840, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 65511} +{"epoch": 0.5625, "grad_norm": 0.5421788096427917, "learning_rate": 8.758017005316988e-05, "loss": 0.003354072803631425, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00336, "step": 144, "tokens/total": 4688200, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 65996} +{"epoch": 0.56640625, "grad_norm": 2.1325883865356445, "learning_rate": 8.737925204046629e-05, "loss": 0.03363441303372383, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.03421, "step": 145, "tokens/total": 4719400, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 66423} +{"epoch": 0.5703125, "grad_norm": 0.5631643533706665, "learning_rate": 8.717698659491851e-05, "loss": 0.02010403200984001, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.02031, "step": 146, "tokens/total": 4753928, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 66905} +{"epoch": 0.57421875, "grad_norm": 8.35096549987793, "learning_rate": 8.697338213361735e-05, "loss": 0.04724488779902458, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.04838, "step": 147, "tokens/total": 4788456, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 67363} +{"epoch": 0.578125, "grad_norm": 0.3780713379383087, "learning_rate": 8.676844712937552e-05, "loss": 0.016445375978946686, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.62, "memory/max_allocated (GiB)": 64.62, "ppl": 1.01658, "step": 148, "tokens/total": 4820096, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 67829} +{"epoch": 0.58203125, "grad_norm": 0.30572086572647095, "learning_rate": 8.656219011037509e-05, "loss": 0.011416308581829071, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.01148, "step": 149, "tokens/total": 4850208, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 68286} +{"epoch": 0.5859375, "grad_norm": 0.37976667284965515, "learning_rate": 8.63546196598125e-05, "loss": 0.006521034520119429, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00654, "step": 150, "tokens/total": 4884720, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 68750} +{"epoch": 0.58984375, "grad_norm": 0.8366066813468933, "learning_rate": 8.614574441554145e-05, "loss": 0.025514084845781326, "memory/device_reserved (GiB)": 66.83, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.02584, "step": 151, "tokens/total": 4922000, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 69231} +{"epoch": 0.59375, "grad_norm": 0.5383761525154114, "learning_rate": 8.593557306971349e-05, "loss": 0.012626614421606064, "memory/device_reserved (GiB)": 66.83, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.01271, "step": 152, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 69711} +{"epoch": 0.59765625, "grad_norm": 0.7415899038314819, "learning_rate": 8.572411436841618e-05, "loss": 0.019816424697637558, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.02001, "step": 153, "tokens/total": 4992880, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 70178} +{"epoch": 0.6015625, "grad_norm": 0.3881523907184601, "learning_rate": 8.551137711130922e-05, "loss": 0.0029186015017330647, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00292, "step": 154, "tokens/total": 5025136, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 70670} +{"epoch": 0.60546875, "grad_norm": 0.5282965898513794, "learning_rate": 8.529737015125824e-05, "loss": 0.0075391060672700405, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00757, "step": 155, "tokens/total": 5058912, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 71111} +{"epoch": 0.609375, "grad_norm": 1.1813163757324219, "learning_rate": 8.508210239396639e-05, "loss": 0.015456758439540863, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.01558, "step": 156, "tokens/total": 5091432, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 71623} +{"epoch": 0.61328125, "grad_norm": 0.4023711681365967, "learning_rate": 8.486558279760375e-05, "loss": 0.0056258006952703, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00564, "step": 157, "tokens/total": 5125920, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 72100} +{"epoch": 0.6171875, "grad_norm": 0.24186982214450836, "learning_rate": 8.464782037243449e-05, "loss": 0.004109546542167664, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00412, "step": 158, "tokens/total": 5159664, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 72598} +{"epoch": 0.62109375, "grad_norm": 0.49737975001335144, "learning_rate": 8.442882418044202e-05, "loss": 0.008957103826105595, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.009, "step": 159, "tokens/total": 5195584, "tokens/train_per_sec_per_gpu": 6.68, "tokens/trainable": 73071} +{"epoch": 0.625, "grad_norm": 0.33527079224586487, "learning_rate": 8.420860333495179e-05, "loss": 0.005354952998459339, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.00537, "step": 160, "tokens/total": 5229664, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 73560} +{"epoch": 0.62890625, "grad_norm": 0.31346866488456726, "learning_rate": 8.398716700025208e-05, "loss": 0.013080188073217869, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.01317, "step": 161, "tokens/total": 5264008, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 74083} +{"epoch": 0.6328125, "grad_norm": 0.46068406105041504, "learning_rate": 8.376452439121266e-05, "loss": 0.011878617107868195, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.01195, "step": 162, "tokens/total": 5294720, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 74530} +{"epoch": 0.63671875, "grad_norm": 0.9991003274917603, "learning_rate": 8.354068477290124e-05, "loss": 0.013094868510961533, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.01318, "step": 163, "tokens/total": 5325376, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 74999} +{"epoch": 0.640625, "grad_norm": 0.627152681350708, "learning_rate": 8.331565746019807e-05, "loss": 0.016897564753890038, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.01704, "step": 164, "tokens/total": 5360240, "tokens/train_per_sec_per_gpu": 6.89, "tokens/trainable": 75450} +{"epoch": 0.64453125, "grad_norm": 0.4373972415924072, "learning_rate": 8.308945181740812e-05, "loss": 0.0057163177989423275, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00573, "step": 165, "tokens/total": 5392376, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 75921} +{"epoch": 0.6484375, "grad_norm": 0.5617805123329163, "learning_rate": 8.286207725787153e-05, "loss": 0.01290193758904934, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.01299, "step": 166, "tokens/total": 5427456, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 76425} +{"epoch": 0.65234375, "grad_norm": 0.5076725482940674, "learning_rate": 8.263354324357182e-05, "loss": 0.0163919348269701, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.01653, "step": 167, "tokens/total": 5457688, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 76863} +{"epoch": 0.65625, "grad_norm": 1.3670915365219116, "learning_rate": 8.240385928474219e-05, "loss": 0.02994699962437153, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.0304, "step": 168, "tokens/total": 5488600, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 77298} +{"epoch": 0.66015625, "grad_norm": 0.300004780292511, "learning_rate": 8.217303493946967e-05, "loss": 0.004375687334686518, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00439, "step": 169, "tokens/total": 5521856, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 77735} +{"epoch": 0.6640625, "grad_norm": 0.5155059099197388, "learning_rate": 8.194107981329746e-05, "loss": 0.006570157594978809, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00659, "step": 170, "tokens/total": 5553272, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 78179} +{"epoch": 0.66796875, "grad_norm": 0.4841272830963135, "learning_rate": 8.170800355882518e-05, "loss": 0.021497154608368874, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.57, "memory/max_allocated (GiB)": 65.57, "ppl": 1.02173, "step": 171, "tokens/total": 5587776, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 78660} +{"epoch": 0.671875, "grad_norm": 0.6374057531356812, "learning_rate": 8.147381587530713e-05, "loss": 0.004224079195410013, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00423, "step": 172, "tokens/total": 5620864, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 79110} +{"epoch": 0.67578125, "grad_norm": 1.5119833946228027, "learning_rate": 8.123852650824877e-05, "loss": 0.017526742070913315, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.01768, "step": 173, "tokens/total": 5653656, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 79576} +{"epoch": 0.6796875, "grad_norm": 0.12675082683563232, "learning_rate": 8.100214524900103e-05, "loss": 0.0013712779618799686, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00137, "step": 174, "tokens/total": 5682312, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 80001} +{"epoch": 0.68359375, "grad_norm": 21.21824836730957, "learning_rate": 8.076468193435301e-05, "loss": 0.017997676506638527, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.01816, "step": 175, "tokens/total": 5713520, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 80452} +{"epoch": 0.6875, "grad_norm": 0.33634403347969055, "learning_rate": 8.052614644612253e-05, "loss": 0.004610471893101931, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.00462, "step": 176, "tokens/total": 5745296, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 80886} +{"epoch": 0.69140625, "grad_norm": 0.46341097354888916, "learning_rate": 8.028654871074489e-05, "loss": 0.012105985544621944, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.01218, "step": 177, "tokens/total": 5774336, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 81325} +{"epoch": 0.6953125, "grad_norm": 1.5226136445999146, "learning_rate": 8.004589869885986e-05, "loss": 0.045225389301776886, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.04626, "step": 178, "tokens/total": 5805032, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 81732} +{"epoch": 0.69921875, "grad_norm": 0.450292706489563, "learning_rate": 7.980420642489674e-05, "loss": 0.013893458992242813, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.01399, "step": 179, "tokens/total": 5832616, "tokens/train_per_sec_per_gpu": 10.95, "tokens/trainable": 82171} +{"epoch": 0.703125, "grad_norm": 1.9392887353897095, "learning_rate": 7.95614819466576e-05, "loss": 0.019606469199061394, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.0198, "step": 180, "tokens/total": 5865568, "tokens/train_per_sec_per_gpu": 10.27, "tokens/trainable": 82670} +{"epoch": 0.70703125, "grad_norm": 0.6367948055267334, "learning_rate": 7.931773536489872e-05, "loss": 0.015185635536909103, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.0153, "step": 181, "tokens/total": 5894256, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 83084} +{"epoch": 0.7109375, "grad_norm": 0.43493449687957764, "learning_rate": 7.907297682291035e-05, "loss": 0.006911782547831535, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00694, "step": 182, "tokens/total": 5924304, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 83500} +{"epoch": 0.71484375, "grad_norm": 0.7213590145111084, "learning_rate": 7.882721650609442e-05, "loss": 0.011483591049909592, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.01155, "step": 183, "tokens/total": 5955128, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 83896} +{"epoch": 0.71875, "grad_norm": 0.488407701253891, "learning_rate": 7.85804646415409e-05, "loss": 0.011025159619748592, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.86, "memory/max_allocated (GiB)": 64.86, "ppl": 1.01109, "step": 184, "tokens/total": 5988056, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 84399} +{"epoch": 0.72265625, "grad_norm": 0.2253665328025818, "learning_rate": 7.833273149760207e-05, "loss": 0.004580401349812746, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00459, "step": 185, "tokens/total": 6019544, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 84873} +{"epoch": 0.7265625, "grad_norm": 0.9831172227859497, "learning_rate": 7.808402738346527e-05, "loss": 0.015935996547341347, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.01606, "step": 186, "tokens/total": 6051616, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 85296} +{"epoch": 0.73046875, "grad_norm": 0.2950233519077301, "learning_rate": 7.783436264872382e-05, "loss": 0.006565968971699476, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.54, "memory/max_allocated (GiB)": 64.54, "ppl": 1.00659, "step": 187, "tokens/total": 6084304, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 85741} +{"epoch": 0.734375, "grad_norm": 0.43974339962005615, "learning_rate": 7.758374768294647e-05, "loss": 0.004686863161623478, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.0047, "step": 188, "tokens/total": 6117896, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 86176} +{"epoch": 0.73828125, "grad_norm": 0.7261695861816406, "learning_rate": 7.733219291524489e-05, "loss": 0.005879863165318966, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.0059, "step": 189, "tokens/total": 6152296, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 86649} +{"epoch": 0.7421875, "grad_norm": 0.5236450433731079, "learning_rate": 7.707970881383977e-05, "loss": 0.015256796032190323, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.01537, "step": 190, "tokens/total": 6186056, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 87084} +{"epoch": 0.74609375, "grad_norm": 1.4506622552871704, "learning_rate": 7.682630588562518e-05, "loss": 0.028806118294596672, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.02923, "step": 191, "tokens/total": 6219160, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 87499} +{"epoch": 0.75, "grad_norm": 0.11594641953706741, "learning_rate": 7.657199467573129e-05, "loss": 0.0006876162369735539, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00069, "step": 192, "tokens/total": 6254840, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 87987} +{"epoch": 0.75390625, "grad_norm": 0.8588489890098572, "learning_rate": 7.631678576708561e-05, "loss": 0.0036344188265502453, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00364, "step": 193, "tokens/total": 6289656, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 88470} +{"epoch": 0.7578125, "grad_norm": 0.02584405615925789, "learning_rate": 7.606068977997255e-05, "loss": 0.00040498393354937434, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00041, "step": 194, "tokens/total": 6321776, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 88923} +{"epoch": 0.76171875, "grad_norm": 0.3917387127876282, "learning_rate": 7.580371737159148e-05, "loss": 0.0009320569224655628, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00093, "step": 195, "tokens/total": 6352824, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 89338} +{"epoch": 0.765625, "grad_norm": 0.36633074283599854, "learning_rate": 7.554587923561324e-05, "loss": 0.004120190627872944, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00413, "step": 196, "tokens/total": 6388496, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 89812} +{"epoch": 0.76953125, "grad_norm": 1.2315809726715088, "learning_rate": 7.528718610173511e-05, "loss": 0.020024297758936882, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.02023, "step": 197, "tokens/total": 6420600, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 90252} +{"epoch": 0.7734375, "grad_norm": 0.40063828229904175, "learning_rate": 7.502764873523431e-05, "loss": 0.004327613860368729, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00434, "step": 198, "tokens/total": 6454616, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 90718} +{"epoch": 0.77734375, "grad_norm": 0.21157366037368774, "learning_rate": 7.476727793652011e-05, "loss": 0.001770398230291903, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00177, "step": 199, "tokens/total": 6485680, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 91194} +{"epoch": 0.78125, "grad_norm": 0.37556561827659607, "learning_rate": 7.450608454068415e-05, "loss": 0.0016035372391343117, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.0016, "step": 200, "tokens/total": 6518512, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 91649} +{"epoch": 0.78515625, "grad_norm": 0.3948712646961212, "learning_rate": 7.424407941704987e-05, "loss": 0.015436811372637749, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.01556, "step": 201, "tokens/total": 6548728, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 92078} +{"epoch": 0.7890625, "grad_norm": 1.228007197380066, "learning_rate": 7.398127346871986e-05, "loss": 0.005694309715181589, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00571, "step": 202, "tokens/total": 6582192, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 92536} +{"epoch": 0.79296875, "grad_norm": 0.37424442172050476, "learning_rate": 7.371767763212238e-05, "loss": 0.009841389022767544, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.00989, "step": 203, "tokens/total": 6617392, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 93032} +{"epoch": 0.796875, "grad_norm": 0.046682704240083694, "learning_rate": 7.345330287655617e-05, "loss": 0.0002635623386595398, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00026, "step": 204, "tokens/total": 6648904, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 93504} +{"epoch": 0.80078125, "grad_norm": 0.41084572672843933, "learning_rate": 7.31881602037339e-05, "loss": 0.003385471645742655, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00339, "step": 205, "tokens/total": 6679832, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 93925} +{"epoch": 0.8046875, "grad_norm": 0.8628517985343933, "learning_rate": 7.29222606473245e-05, "loss": 0.006332047749310732, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00635, "step": 206, "tokens/total": 6710784, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 94365} +{"epoch": 0.80859375, "grad_norm": 0.3156498372554779, "learning_rate": 7.265561527249383e-05, "loss": 0.0015351248439401388, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00154, "step": 207, "tokens/total": 6744848, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 94845} +{"epoch": 0.8125, "grad_norm": 0.09315948188304901, "learning_rate": 7.238823517544436e-05, "loss": 0.0012833857908844948, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00128, "step": 208, "tokens/total": 6777600, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 95285} +{"epoch": 0.81640625, "grad_norm": 0.050574056804180145, "learning_rate": 7.212013148295333e-05, "loss": 0.0006550142425112426, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00066, "step": 209, "tokens/total": 6811024, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 95758} +{"epoch": 0.8203125, "grad_norm": 1.3590624332427979, "learning_rate": 7.185131535190975e-05, "loss": 0.005826662294566631, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.15, "memory/max_allocated (GiB)": 66.15, "ppl": 1.00584, "step": 210, "tokens/total": 6845376, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 96192} +{"epoch": 0.82421875, "grad_norm": 0.6694889068603516, "learning_rate": 7.158179796885005e-05, "loss": 0.010391799733042717, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01045, "step": 211, "tokens/total": 6880776, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 96641} +{"epoch": 0.828125, "grad_norm": 0.792132556438446, "learning_rate": 7.131159054949273e-05, "loss": 0.008358408696949482, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00839, "step": 212, "tokens/total": 6916160, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 97160} +{"epoch": 0.83203125, "grad_norm": 0.049760766327381134, "learning_rate": 7.104070433827139e-05, "loss": 0.000576911261305213, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.54, "memory/max_allocated (GiB)": 63.54, "ppl": 1.00058, "step": 213, "tokens/total": 6945320, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 97611} +{"epoch": 0.8359375, "grad_norm": 0.6148718595504761, "learning_rate": 7.076915060786705e-05, "loss": 0.008072498254477978, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00811, "step": 214, "tokens/total": 6977856, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 98111} +{"epoch": 0.83984375, "grad_norm": 0.11600159108638763, "learning_rate": 7.049694065873882e-05, "loss": 0.0007807848160155118, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.00078, "step": 215, "tokens/total": 7011096, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 98575} +{"epoch": 0.84375, "grad_norm": 0.6265577673912048, "learning_rate": 7.022408581865382e-05, "loss": 0.024433132261037827, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.93, "memory/max_allocated (GiB)": 63.93, "ppl": 1.02473, "step": 216, "tokens/total": 7041408, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 98991} +{"epoch": 0.84765625, "grad_norm": 0.5053733587265015, "learning_rate": 6.99505974422157e-05, "loss": 0.004164066165685654, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.6, "memory/max_allocated (GiB)": 63.6, "ppl": 1.00417, "step": 217, "tokens/total": 7071560, "tokens/train_per_sec_per_gpu": 11.33, "tokens/trainable": 99454} +{"epoch": 0.8515625, "grad_norm": 0.029268058016896248, "learning_rate": 6.967648691039213e-05, "loss": 0.0003608351107686758, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00036, "step": 218, "tokens/total": 7104696, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 99925} +{"epoch": 0.85546875, "grad_norm": 0.1441776603460312, "learning_rate": 6.940176563004123e-05, "loss": 0.0012545227073132992, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00126, "step": 219, "tokens/total": 7138504, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 100371} +{"epoch": 0.859375, "grad_norm": 0.6769626140594482, "learning_rate": 6.912644503343682e-05, "loss": 0.009973976761102676, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.01002, "step": 220, "tokens/total": 7170992, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 100855} +{"epoch": 0.86328125, "grad_norm": 0.8816158175468445, "learning_rate": 6.885053657779273e-05, "loss": 0.014406959526240826, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.01451, "step": 221, "tokens/total": 7201176, "tokens/train_per_sec_per_gpu": 6.76, "tokens/trainable": 101253} +{"epoch": 0.8671875, "grad_norm": 0.23672012984752655, "learning_rate": 6.857405174478604e-05, "loss": 0.004113221541047096, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00412, "step": 222, "tokens/total": 7233528, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 101706} +{"epoch": 0.87109375, "grad_norm": 0.05168158560991287, "learning_rate": 6.82970020400792e-05, "loss": 0.0007311180816031992, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00073, "step": 223, "tokens/total": 7267672, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 102158} +{"epoch": 0.875, "grad_norm": 0.7718090415000916, "learning_rate": 6.801939899284132e-05, "loss": 0.011144733056426048, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 62.86, "memory/max_allocated (GiB)": 62.86, "ppl": 1.01121, "step": 224, "tokens/total": 7295592, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 102580} +{"epoch": 0.87890625, "grad_norm": 0.2375320941209793, "learning_rate": 6.774125415526827e-05, "loss": 0.0019279668340459466, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00193, "step": 225, "tokens/total": 7325816, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 103063} +{"epoch": 0.8828125, "grad_norm": 0.13583868741989136, "learning_rate": 6.746257910210214e-05, "loss": 0.0011997056426480412, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.0012, "step": 226, "tokens/total": 7360888, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 103510} +{"epoch": 0.88671875, "grad_norm": 0.49465298652648926, "learning_rate": 6.718338543014937e-05, "loss": 0.00565455574542284, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00567, "step": 227, "tokens/total": 7390416, "tokens/train_per_sec_per_gpu": 10.1, "tokens/trainable": 103980} +{"epoch": 0.890625, "grad_norm": 0.10906943678855896, "learning_rate": 6.69036847577983e-05, "loss": 0.0005099020781926811, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00051, "step": 228, "tokens/total": 7420480, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 104413} +{"epoch": 0.89453125, "grad_norm": 0.07020265609025955, "learning_rate": 6.662348872453553e-05, "loss": 0.0003678625216707587, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00037, "step": 229, "tokens/total": 7448992, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 104845} +{"epoch": 0.8984375, "grad_norm": 0.06628313660621643, "learning_rate": 6.63428089904618e-05, "loss": 0.0007140293600969017, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00071, "step": 230, "tokens/total": 7481744, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 105289} +{"epoch": 0.90234375, "grad_norm": 1.2308144569396973, "learning_rate": 6.60616572358065e-05, "loss": 0.007536487653851509, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00756, "step": 231, "tokens/total": 7516704, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 105722} +{"epoch": 0.90625, "grad_norm": 0.07671478390693665, "learning_rate": 6.578004516044172e-05, "loss": 0.0006240092916414142, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.19, "memory/max_allocated (GiB)": 64.19, "ppl": 1.00062, "step": 232, "tokens/total": 7548592, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 106213} +{"epoch": 0.91015625, "grad_norm": 0.48954856395721436, "learning_rate": 6.549798448339548e-05, "loss": 0.004460042342543602, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00447, "step": 233, "tokens/total": 7579288, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 106643} +{"epoch": 0.9140625, "grad_norm": 0.36759454011917114, "learning_rate": 6.521548694236384e-05, "loss": 0.0032748300582170486, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00328, "step": 234, "tokens/total": 7611672, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 107136} +{"epoch": 0.91796875, "grad_norm": 0.4002198278903961, "learning_rate": 6.493256429322259e-05, "loss": 0.002622877014800906, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00263, "step": 235, "tokens/total": 7646288, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 107588} +{"epoch": 0.921875, "grad_norm": 1.01121985912323, "learning_rate": 6.464922830953799e-05, "loss": 0.027408523485064507, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.02779, "step": 236, "tokens/total": 7680136, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 108087} +{"epoch": 0.92578125, "grad_norm": 0.12471899390220642, "learning_rate": 6.436549078207688e-05, "loss": 0.0007101054070517421, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.61, "memory/max_allocated (GiB)": 64.61, "ppl": 1.00071, "step": 237, "tokens/total": 7713072, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 108533} +{"epoch": 0.9296875, "grad_norm": 0.08625931292772293, "learning_rate": 6.408136351831592e-05, "loss": 0.001185489585623145, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00119, "step": 238, "tokens/total": 7747856, "tokens/train_per_sec_per_gpu": 6.91, "tokens/trainable": 108965} +{"epoch": 0.93359375, "grad_norm": 0.36999648809432983, "learning_rate": 6.379685834195036e-05, "loss": 0.005553287919610739, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00557, "step": 239, "tokens/total": 7780136, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 109416} +{"epoch": 0.9375, "grad_norm": 0.06372952461242676, "learning_rate": 6.351198709240186e-05, "loss": 0.0008587866323068738, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.78, "memory/max_allocated (GiB)": 65.78, "ppl": 1.00086, "step": 240, "tokens/total": 7816344, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 109864} +{"epoch": 0.94140625, "grad_norm": 0.4588616192340851, "learning_rate": 6.32267616243259e-05, "loss": 0.002562975976616144, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00257, "step": 241, "tokens/total": 7850408, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 110345} +{"epoch": 0.9453125, "grad_norm": 0.2275570034980774, "learning_rate": 6.294119380711849e-05, "loss": 0.00032354649738408625, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00032, "step": 242, "tokens/total": 7881528, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 110794} +{"epoch": 0.94921875, "grad_norm": 0.04456785321235657, "learning_rate": 6.265529552442209e-05, "loss": 0.0007506685797125101, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00075, "step": 243, "tokens/total": 7915712, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 111250} +{"epoch": 0.953125, "grad_norm": 0.597923219203949, "learning_rate": 6.236907867363127e-05, "loss": 0.008774288929998875, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.61, "memory/max_allocated (GiB)": 64.61, "ppl": 1.00881, "step": 244, "tokens/total": 7947408, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 111693} +{"epoch": 0.95703125, "grad_norm": 0.6968534588813782, "learning_rate": 6.208255516539749e-05, "loss": 0.01229805126786232, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.26, "memory/max_allocated (GiB)": 65.26, "ppl": 1.01237, "step": 245, "tokens/total": 7979600, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 112158} +{"epoch": 0.9609375, "grad_norm": 0.3104414939880371, "learning_rate": 6.179573692313344e-05, "loss": 0.007209367584437132, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00724, "step": 246, "tokens/total": 8010848, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 112601} +{"epoch": 0.96484375, "grad_norm": 0.12009605765342712, "learning_rate": 6.150863588251694e-05, "loss": 0.0019254572689533234, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.00193, "step": 247, "tokens/total": 8040632, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 113016} +{"epoch": 0.96875, "grad_norm": 0.07025254517793655, "learning_rate": 6.122126399099419e-05, "loss": 0.0005987431504763663, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.0006, "step": 248, "tokens/total": 8072464, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 113476} +{"epoch": 0.97265625, "grad_norm": 0.6118393540382385, "learning_rate": 6.0933633207282615e-05, "loss": 0.011351184919476509, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.96, "memory/max_allocated (GiB)": 64.96, "ppl": 1.01142, "step": 249, "tokens/total": 8106568, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 113941} +{"epoch": 0.9765625, "grad_norm": 0.621365487575531, "learning_rate": 6.064575550087316e-05, "loss": 0.023094069212675095, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.02336, "step": 250, "tokens/total": 8138496, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 114434} +{"epoch": 0.98046875, "grad_norm": 0.11961700022220612, "learning_rate": 6.0357642851532245e-05, "loss": 0.0019762562587857246, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00198, "step": 251, "tokens/total": 8168080, "tokens/train_per_sec_per_gpu": 10.22, "tokens/trainable": 114898} +{"epoch": 0.984375, "grad_norm": 0.08516581356525421, "learning_rate": 6.0069307248803294e-05, "loss": 0.0010312879458069801, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00103, "step": 252, "tokens/total": 8200888, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 115331} +{"epoch": 0.98828125, "grad_norm": 0.041669610887765884, "learning_rate": 5.9780760691507635e-05, "loss": 0.0006709538865834475, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00067, "step": 253, "tokens/total": 8230704, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 115732} +{"epoch": 0.9921875, "grad_norm": 0.14642614126205444, "learning_rate": 5.9492015187245334e-05, "loss": 0.004409186076372862, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00442, "step": 254, "tokens/total": 8266560, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 116221} +{"epoch": 0.99609375, "grad_norm": 0.10179398953914642, "learning_rate": 5.920308275189541e-05, "loss": 0.002153647132217884, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.19, "memory/max_allocated (GiB)": 65.19, "ppl": 1.00216, "step": 255, "tokens/total": 8300184, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 116684} +{"epoch": 1.0, "grad_norm": 0.06430695205926895, "learning_rate": 5.8913975409115874e-05, "loss": 0.00041164428694173694, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00041, "step": 256, "tokens/total": 8333624, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 117126} +{"epoch": 1.00390625, "grad_norm": 0.10967746376991272, "learning_rate": 5.8624705189843395e-05, "loss": 0.0012721186503767967, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00127, "step": 257, "tokens/total": 8364976, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 117592} +{"epoch": 1.0078125, "grad_norm": 0.05266302451491356, "learning_rate": 5.833528413179249e-05, "loss": 0.0009076150599867105, "memory/device_reserved (GiB)": 66.62, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00091, "step": 258, "tokens/total": 8399568, "tokens/train_per_sec_per_gpu": 6.95, "tokens/trainable": 118030} +{"epoch": 1.01171875, "grad_norm": 0.2797621488571167, "learning_rate": 5.80457242789548e-05, "loss": 0.00399803277105093, "memory/device_reserved (GiB)": 66.62, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00401, "step": 259, "tokens/total": 8430616, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 118470} +{"epoch": 1.015625, "grad_norm": 0.5181891918182373, "learning_rate": 5.77560376810977e-05, "loss": 0.012039871886372566, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.01211, "step": 260, "tokens/total": 8462696, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 118935} +{"epoch": 1.01953125, "grad_norm": 0.2075675129890442, "learning_rate": 5.7466236393263005e-05, "loss": 0.002205880591645837, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00221, "step": 261, "tokens/total": 8496152, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 119381} +{"epoch": 1.0234375, "grad_norm": 1.3273930549621582, "learning_rate": 5.717633247526522e-05, "loss": 0.0014501737896353006, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00145, "step": 262, "tokens/total": 8532912, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 119849} +{"epoch": 1.02734375, "grad_norm": 0.08663095533847809, "learning_rate": 5.688633799118971e-05, "loss": 0.0005788251291960478, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.00058, "step": 263, "tokens/total": 8565376, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 120319} +{"epoch": 1.03125, "grad_norm": 0.10334321111440659, "learning_rate": 5.659626500889066e-05, "loss": 0.0010221308330073953, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00102, "step": 264, "tokens/total": 8597640, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 120804} +{"epoch": 1.03515625, "grad_norm": 0.02445986680686474, "learning_rate": 5.6306125599488905e-05, "loss": 0.00022592479945160449, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.00023, "step": 265, "tokens/total": 8627312, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 121233} +{"epoch": 1.0390625, "grad_norm": 0.07822425663471222, "learning_rate": 5.601593183686955e-05, "loss": 0.00046964274952188134, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.87, "memory/max_allocated (GiB)": 64.87, "ppl": 1.00047, "step": 266, "tokens/total": 8659208, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 121671} +{"epoch": 1.04296875, "grad_norm": 0.16447320580482483, "learning_rate": 5.572569579717961e-05, "loss": 0.0010317087871953845, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.00103, "step": 267, "tokens/total": 8693088, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 122141} +{"epoch": 1.046875, "grad_norm": 0.22877687215805054, "learning_rate": 5.543542955832538e-05, "loss": 0.0018124886555597186, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00181, "step": 268, "tokens/total": 8725576, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 122556} +{"epoch": 1.05078125, "grad_norm": 0.08380807936191559, "learning_rate": 5.514514519946986e-05, "loss": 0.0006176539463922381, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00062, "step": 269, "tokens/total": 8755192, "tokens/train_per_sec_per_gpu": 10.3, "tokens/trainable": 122988} +{"epoch": 1.0546875, "grad_norm": 0.0029943448025733232, "learning_rate": 5.485485480053015e-05, "loss": 2.704598955460824e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00003, "step": 270, "tokens/total": 8787704, "tokens/train_per_sec_per_gpu": 10.52, "tokens/trainable": 123478} +{"epoch": 1.05859375, "grad_norm": 0.423817902803421, "learning_rate": 5.4564570441674645e-05, "loss": 0.0019342132145538926, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00194, "step": 271, "tokens/total": 8819768, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 123932} +{"epoch": 1.0625, "grad_norm": 0.2672625482082367, "learning_rate": 5.42743042028204e-05, "loss": 0.0020168409682810307, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00202, "step": 272, "tokens/total": 8853352, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 124428} +{"epoch": 1.06640625, "grad_norm": 0.07779988646507263, "learning_rate": 5.3984068163130464e-05, "loss": 0.0003732169861905277, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00037, "step": 273, "tokens/total": 8886840, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 124866} +{"epoch": 1.0703125, "grad_norm": 0.06881003826856613, "learning_rate": 5.369387440051111e-05, "loss": 0.0004991627065464854, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.0005, "step": 274, "tokens/total": 8920808, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 125324} +{"epoch": 1.07421875, "grad_norm": 0.3793438971042633, "learning_rate": 5.340373499110935e-05, "loss": 0.008153737522661686, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00819, "step": 275, "tokens/total": 8955880, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 125813} +{"epoch": 1.078125, "grad_norm": 0.4732087254524231, "learning_rate": 5.3113662008810304e-05, "loss": 0.00257208663970232, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.00258, "step": 276, "tokens/total": 8989520, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 126252} +{"epoch": 1.08203125, "grad_norm": 0.008997482247650623, "learning_rate": 5.282366752473479e-05, "loss": 8.634535333840176e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00009, "step": 277, "tokens/total": 9024680, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 126696} +{"epoch": 1.0859375, "grad_norm": 0.9168142676353455, "learning_rate": 5.2533763606737005e-05, "loss": 0.006826003082096577, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00685, "step": 278, "tokens/total": 9056208, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 127173} +{"epoch": 1.08984375, "grad_norm": 0.06601894646883011, "learning_rate": 5.224396231890232e-05, "loss": 9.999227040680125e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.0001, "step": 279, "tokens/total": 9086376, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 127637} +{"epoch": 1.09375, "grad_norm": 0.002862143563106656, "learning_rate": 5.195427572104522e-05, "loss": 2.0933663108735345e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00002, "step": 280, "tokens/total": 9118944, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 128112} +{"epoch": 1.09765625, "grad_norm": 0.05245296284556389, "learning_rate": 5.166471586820751e-05, "loss": 0.0003029380750376731, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.0003, "step": 281, "tokens/total": 9152920, "tokens/train_per_sec_per_gpu": 7.0, "tokens/trainable": 128584} +{"epoch": 1.1015625, "grad_norm": 0.07506691664457321, "learning_rate": 5.1375294810156615e-05, "loss": 0.0002442966215312481, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00024, "step": 282, "tokens/total": 9185728, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 129051} +{"epoch": 1.10546875, "grad_norm": 0.005588609725236893, "learning_rate": 5.1086024590884144e-05, "loss": 4.408282620715909e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00004, "step": 283, "tokens/total": 9217984, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 129494} +{"epoch": 1.109375, "grad_norm": 0.03318503126502037, "learning_rate": 5.079691724810461e-05, "loss": 0.00019155786139890552, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00019, "step": 284, "tokens/total": 9253160, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 129983} +{"epoch": 1.11328125, "grad_norm": 0.03128315880894661, "learning_rate": 5.0507984812754684e-05, "loss": 0.0004197126836515963, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00042, "step": 285, "tokens/total": 9286136, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 130405} +{"epoch": 1.1171875, "grad_norm": 0.014215673319995403, "learning_rate": 5.021923930849237e-05, "loss": 7.688493496971205e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00008, "step": 286, "tokens/total": 9318360, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 130829} +{"epoch": 1.12109375, "grad_norm": 0.4045202434062958, "learning_rate": 4.99306927511967e-05, "loss": 0.005534152965992689, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00555, "step": 287, "tokens/total": 9350288, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 131287} +{"epoch": 1.125, "grad_norm": 0.20825296640396118, "learning_rate": 4.964235714846775e-05, "loss": 0.0008888263837434351, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00089, "step": 288, "tokens/total": 9382104, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 131712} +{"epoch": 1.12890625, "grad_norm": 0.2577660381793976, "learning_rate": 4.9354244499126866e-05, "loss": 0.0014218362048268318, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00142, "step": 289, "tokens/total": 9414392, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 132168} +{"epoch": 1.1328125, "grad_norm": 0.07468997687101364, "learning_rate": 4.90663667927174e-05, "loss": 0.0005830723093822598, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00058, "step": 290, "tokens/total": 9447112, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 132626} +{"epoch": 1.13671875, "grad_norm": 0.9964237213134766, "learning_rate": 4.877873600900581e-05, "loss": 0.007051088847219944, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00708, "step": 291, "tokens/total": 9483400, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 133119} +{"epoch": 1.140625, "grad_norm": 0.00119964184705168, "learning_rate": 4.849136411748306e-05, "loss": 1.4345479030453134e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.48, "memory/max_allocated (GiB)": 64.48, "ppl": 1.00001, "step": 292, "tokens/total": 9513896, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 133550} +{"epoch": 1.14453125, "grad_norm": 0.9379732608795166, "learning_rate": 4.8204263076866574e-05, "loss": 0.007233927026391029, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00726, "step": 293, "tokens/total": 9550184, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 133992} +{"epoch": 1.1484375, "grad_norm": 0.17889894545078278, "learning_rate": 4.791744483460251e-05, "loss": 0.0006848265184089541, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00069, "step": 294, "tokens/total": 9583920, "tokens/train_per_sec_per_gpu": 10.07, "tokens/trainable": 134467} +{"epoch": 1.15234375, "grad_norm": 0.2112904191017151, "learning_rate": 4.7630921326368736e-05, "loss": 0.000541175773832947, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00054, "step": 295, "tokens/total": 9617288, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 134895} +{"epoch": 1.15625, "grad_norm": 0.07525084912776947, "learning_rate": 4.7344704475577916e-05, "loss": 0.0003049848019145429, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 62.94, "memory/max_allocated (GiB)": 62.94, "ppl": 1.00031, "step": 296, "tokens/total": 9645600, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 135321} +{"epoch": 1.16015625, "grad_norm": 0.7428085207939148, "learning_rate": 4.705880619288153e-05, "loss": 0.020198147743940353, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.0204, "step": 297, "tokens/total": 9679456, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 135765} +{"epoch": 1.1640625, "grad_norm": 0.03754932060837746, "learning_rate": 4.677323837567412e-05, "loss": 0.00017894413031172007, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00018, "step": 298, "tokens/total": 9711696, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 136196} +{"epoch": 1.16796875, "grad_norm": 0.18205712735652924, "learning_rate": 4.6488012907598146e-05, "loss": 0.00041266431799158454, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00041, "step": 299, "tokens/total": 9745792, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 136649} +{"epoch": 1.171875, "grad_norm": 0.0370117649435997, "learning_rate": 4.620314165804964e-05, "loss": 0.0003721847024280578, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00037, "step": 300, "tokens/total": 9780648, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 137128} +{"epoch": 1.17578125, "grad_norm": 0.04210005700588226, "learning_rate": 4.591863648168407e-05, "loss": 0.0006169796106405556, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00062, "step": 301, "tokens/total": 9813032, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 137585} +{"epoch": 1.1796875, "grad_norm": 0.9307562708854675, "learning_rate": 4.5634509217923135e-05, "loss": 0.008932534605264664, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00897, "step": 302, "tokens/total": 9845200, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 138054} +{"epoch": 1.18359375, "grad_norm": 0.04537074640393257, "learning_rate": 4.535077169046201e-05, "loss": 0.0003396293905097991, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.29, "memory/max_allocated (GiB)": 65.29, "ppl": 1.00034, "step": 303, "tokens/total": 9878616, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 138559} +{"epoch": 1.1875, "grad_norm": 0.1241551861166954, "learning_rate": 4.506743570677743e-05, "loss": 0.0009222638327628374, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00092, "step": 304, "tokens/total": 9910776, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 139050} +{"epoch": 1.19140625, "grad_norm": 0.02856675535440445, "learning_rate": 4.478451305763618e-05, "loss": 0.0002343205560464412, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00023, "step": 305, "tokens/total": 9945480, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 139525} +{"epoch": 1.1953125, "grad_norm": 0.09989161044359207, "learning_rate": 4.450201551660454e-05, "loss": 0.0004777932190336287, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.00048, "step": 306, "tokens/total": 9975992, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 139969} +{"epoch": 1.19921875, "grad_norm": 0.15369859337806702, "learning_rate": 4.4219954839558276e-05, "loss": 0.0007721863803453743, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.00077, "step": 307, "tokens/total": 10006440, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 140443} +{"epoch": 1.203125, "grad_norm": 0.009717502631247044, "learning_rate": 4.393834276419352e-05, "loss": 8.548731420887634e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00009, "step": 308, "tokens/total": 10042928, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 140926} +{"epoch": 1.20703125, "grad_norm": 0.003392493585124612, "learning_rate": 4.36571910095382e-05, "loss": 5.720093395211734e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00006, "step": 309, "tokens/total": 10075184, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 141344} +{"epoch": 1.2109375, "grad_norm": 0.04722871631383896, "learning_rate": 4.337651127546448e-05, "loss": 0.0005675980937667191, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00057, "step": 310, "tokens/total": 10106656, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 141753} +{"epoch": 1.21484375, "grad_norm": 0.09193476289510727, "learning_rate": 4.3096315242201736e-05, "loss": 0.000624972686637193, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.82, "memory/max_allocated (GiB)": 63.82, "ppl": 1.00063, "step": 311, "tokens/total": 10138536, "tokens/train_per_sec_per_gpu": 10.05, "tokens/trainable": 142219} +{"epoch": 1.21875, "grad_norm": 0.8405691385269165, "learning_rate": 4.2816614569850635e-05, "loss": 0.010653275065124035, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.01071, "step": 312, "tokens/total": 10165768, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 142617} +{"epoch": 1.22265625, "grad_norm": 0.17606185376644135, "learning_rate": 4.2537420897897864e-05, "loss": 0.0016231090994551778, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00162, "step": 313, "tokens/total": 10201624, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 143103} +{"epoch": 1.2265625, "grad_norm": 0.13288655877113342, "learning_rate": 4.225874584473174e-05, "loss": 0.0005790909635834396, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00058, "step": 314, "tokens/total": 10233304, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 143535} +{"epoch": 1.23046875, "grad_norm": 0.09101114422082901, "learning_rate": 4.19806010071587e-05, "loss": 0.0010684940498322248, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00107, "step": 315, "tokens/total": 10267904, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 144009} +{"epoch": 1.234375, "grad_norm": 1.4585027694702148, "learning_rate": 4.170299795992081e-05, "loss": 0.022459564730525017, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.02271, "step": 316, "tokens/total": 10301280, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 144460} +{"epoch": 1.23828125, "grad_norm": 0.3438749611377716, "learning_rate": 4.142594825521398e-05, "loss": 0.006246697623282671, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00627, "step": 317, "tokens/total": 10337072, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 144936} +{"epoch": 1.2421875, "grad_norm": 0.07597950845956802, "learning_rate": 4.114946342220728e-05, "loss": 0.0010231725173071027, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.69, "memory/max_allocated (GiB)": 64.69, "ppl": 1.00102, "step": 318, "tokens/total": 10368688, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 145375} +{"epoch": 1.24609375, "grad_norm": 0.021846063435077667, "learning_rate": 4.087355496656321e-05, "loss": 0.00026549541507847607, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00027, "step": 319, "tokens/total": 10400752, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 145849} +{"epoch": 1.25, "grad_norm": 0.01799745485186577, "learning_rate": 4.05982343699588e-05, "loss": 0.0002463912533130497, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00025, "step": 320, "tokens/total": 10432112, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 146318} +{"epoch": 1.25390625, "grad_norm": 0.017100775614380836, "learning_rate": 4.0323513089607876e-05, "loss": 0.0001791166141629219, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00018, "step": 321, "tokens/total": 10465472, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 146821} +{"epoch": 1.2578125, "grad_norm": 0.06104249134659767, "learning_rate": 4.004940255778431e-05, "loss": 0.0006023756577633321, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.0006, "step": 322, "tokens/total": 10496336, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 147285} +{"epoch": 1.26171875, "grad_norm": 0.03441144526004791, "learning_rate": 3.977591418134619e-05, "loss": 0.00028665977879427373, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00029, "step": 323, "tokens/total": 10525864, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 147729} +{"epoch": 1.265625, "grad_norm": 0.9932041168212891, "learning_rate": 3.95030593412612e-05, "loss": 0.020635753870010376, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.02085, "step": 324, "tokens/total": 10559680, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 148169} +{"epoch": 1.26953125, "grad_norm": 0.0660858154296875, "learning_rate": 3.923084939213296e-05, "loss": 0.0007848493405617774, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00079, "step": 325, "tokens/total": 10595400, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 148630} +{"epoch": 1.2734375, "grad_norm": 0.20234271883964539, "learning_rate": 3.895929566172861e-05, "loss": 0.00123718217946589, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00124, "step": 326, "tokens/total": 10626680, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 149063} +{"epoch": 1.27734375, "grad_norm": 0.0380142480134964, "learning_rate": 3.868840945050728e-05, "loss": 0.000369958026567474, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00037, "step": 327, "tokens/total": 10661784, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 149549} +{"epoch": 1.28125, "grad_norm": 0.26147881150245667, "learning_rate": 3.841820203114995e-05, "loss": 0.011334290727972984, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.31, "memory/max_allocated (GiB)": 65.31, "ppl": 1.0114, "step": 328, "tokens/total": 10696184, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 150015} +{"epoch": 1.28515625, "grad_norm": 0.06708770990371704, "learning_rate": 3.814868464809027e-05, "loss": 0.0006567485979758203, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00066, "step": 329, "tokens/total": 10729376, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 150514} +{"epoch": 1.2890625, "grad_norm": 0.3122485876083374, "learning_rate": 3.787986851704667e-05, "loss": 0.002321115927770734, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00232, "step": 330, "tokens/total": 10764824, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 151021} +{"epoch": 1.29296875, "grad_norm": 0.05048302188515663, "learning_rate": 3.7611764824555654e-05, "loss": 0.0007658598478883505, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00077, "step": 331, "tokens/total": 10797072, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 151486} +{"epoch": 1.296875, "grad_norm": 0.0595385879278183, "learning_rate": 3.734438472750619e-05, "loss": 0.0007391087710857391, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00074, "step": 332, "tokens/total": 10829376, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 151956} +{"epoch": 1.30078125, "grad_norm": 0.19063521921634674, "learning_rate": 3.707773935267552e-05, "loss": 0.0009132729610428214, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00091, "step": 333, "tokens/total": 10863512, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 152442} +{"epoch": 1.3046875, "grad_norm": 0.11108580231666565, "learning_rate": 3.68118397962661e-05, "loss": 0.0014792568981647491, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00148, "step": 334, "tokens/total": 10894432, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 152890} +{"epoch": 1.30859375, "grad_norm": 0.3416348397731781, "learning_rate": 3.654669712344384e-05, "loss": 0.004162979777902365, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.34, "memory/max_allocated (GiB)": 63.34, "ppl": 1.00417, "step": 335, "tokens/total": 10924848, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 153368} +{"epoch": 1.3125, "grad_norm": 0.052113600075244904, "learning_rate": 3.628232236787763e-05, "loss": 0.00038639316335320473, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00039, "step": 336, "tokens/total": 10959024, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 153809} +{"epoch": 1.31640625, "grad_norm": 1.1539075374603271, "learning_rate": 3.6018726531280144e-05, "loss": 0.009729829616844654, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00978, "step": 337, "tokens/total": 10991088, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 154258} +{"epoch": 1.3203125, "grad_norm": 0.6585971117019653, "learning_rate": 3.575592058295017e-05, "loss": 0.00352085311897099, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00353, "step": 338, "tokens/total": 11023904, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 154704} +{"epoch": 1.32421875, "grad_norm": 0.028066307306289673, "learning_rate": 3.549391545931585e-05, "loss": 0.0005235387943685055, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00052, "step": 339, "tokens/total": 11057680, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 155157} +{"epoch": 1.328125, "grad_norm": 0.4084787964820862, "learning_rate": 3.5232722063479914e-05, "loss": 0.005509922746568918, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00553, "step": 340, "tokens/total": 11089800, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 155582} +{"epoch": 1.33203125, "grad_norm": 0.012948230840265751, "learning_rate": 3.49723512647657e-05, "loss": 0.0001873790315585211, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00019, "step": 341, "tokens/total": 11117960, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 156047} +{"epoch": 1.3359375, "grad_norm": 0.12517006695270538, "learning_rate": 3.471281389826491e-05, "loss": 0.0016524221282452345, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00165, "step": 342, "tokens/total": 11151888, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 156500} +{"epoch": 1.33984375, "grad_norm": 0.079105906188488, "learning_rate": 3.4454120764386764e-05, "loss": 0.001009129686281085, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.22, "memory/max_allocated (GiB)": 65.22, "ppl": 1.00101, "step": 343, "tokens/total": 11184568, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 156931} +{"epoch": 1.34375, "grad_norm": 0.027443429455161095, "learning_rate": 3.4196282628408526e-05, "loss": 0.00047922326484695077, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00048, "step": 344, "tokens/total": 11216400, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 157385} +{"epoch": 1.34765625, "grad_norm": 0.018244734033942223, "learning_rate": 3.3939310220027456e-05, "loss": 0.00018069040379486978, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00018, "step": 345, "tokens/total": 11250768, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 157817} +{"epoch": 1.3515625, "grad_norm": 0.020322104915976524, "learning_rate": 3.3683214232914404e-05, "loss": 0.00044357747538015246, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00044, "step": 346, "tokens/total": 11281808, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 158273} +{"epoch": 1.35546875, "grad_norm": 0.08280880004167557, "learning_rate": 3.342800532426873e-05, "loss": 0.0009935061680153012, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00099, "step": 347, "tokens/total": 11313968, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 158728} +{"epoch": 1.359375, "grad_norm": 0.008065028116106987, "learning_rate": 3.317369411437484e-05, "loss": 6.599077460123226e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00007, "step": 348, "tokens/total": 11347288, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 159168} +{"epoch": 1.36328125, "grad_norm": 0.007253427058458328, "learning_rate": 3.292029118616024e-05, "loss": 0.00011311725393170491, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00011, "step": 349, "tokens/total": 11377128, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 159592} +{"epoch": 1.3671875, "grad_norm": 0.00982465036213398, "learning_rate": 3.266780708475511e-05, "loss": 0.0001215156153193675, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00012, "step": 350, "tokens/total": 11409488, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 160093} +{"epoch": 1.37109375, "grad_norm": 0.019967420026659966, "learning_rate": 3.241625231705354e-05, "loss": 0.0003664274699985981, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00037, "step": 351, "tokens/total": 11442040, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 160534} +{"epoch": 1.375, "grad_norm": 0.5266655087471008, "learning_rate": 3.216563735127618e-05, "loss": 0.006938511505723, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00696, "step": 352, "tokens/total": 11478312, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 161009} +{"epoch": 1.37890625, "grad_norm": 0.20813347399234772, "learning_rate": 3.191597261653475e-05, "loss": 0.002092310693114996, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.93, "memory/max_allocated (GiB)": 63.93, "ppl": 1.00209, "step": 353, "tokens/total": 11509008, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 161450} +{"epoch": 1.3828125, "grad_norm": 0.14316990971565247, "learning_rate": 3.166726850239794e-05, "loss": 0.0013078817864879966, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00131, "step": 354, "tokens/total": 11541264, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 161900} +{"epoch": 1.38671875, "grad_norm": 0.0025154673494398594, "learning_rate": 3.141953535845912e-05, "loss": 3.521522012306377e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00004, "step": 355, "tokens/total": 11573072, "tokens/train_per_sec_per_gpu": 10.06, "tokens/trainable": 162358} +{"epoch": 1.390625, "grad_norm": 0.6716820597648621, "learning_rate": 3.11727834939056e-05, "loss": 0.00426530884578824, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00427, "step": 356, "tokens/total": 11605224, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 162845} +{"epoch": 1.39453125, "grad_norm": 0.3444748818874359, "learning_rate": 3.092702317708967e-05, "loss": 0.0018079166766256094, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00181, "step": 357, "tokens/total": 11637512, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 163299} +{"epoch": 1.3984375, "grad_norm": 0.02799101732671261, "learning_rate": 3.0682264635101276e-05, "loss": 0.0003573771973606199, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00036, "step": 358, "tokens/total": 11671280, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 163774} +{"epoch": 1.40234375, "grad_norm": 0.20588020980358124, "learning_rate": 3.0438518053342407e-05, "loss": 0.0009896066039800644, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00099, "step": 359, "tokens/total": 11702008, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 164199} +{"epoch": 1.40625, "grad_norm": 0.08815610408782959, "learning_rate": 3.0195793575103266e-05, "loss": 0.0010449958499521017, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00105, "step": 360, "tokens/total": 11732032, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 164667} +{"epoch": 1.41015625, "grad_norm": 0.09919760376214981, "learning_rate": 2.9954101301140146e-05, "loss": 0.001016758382320404, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00102, "step": 361, "tokens/total": 11767472, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.031020376831293106, "learning_rate": 2.9713451289255123e-05, "loss": 0.0003684962575789541, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00037, "step": 362, "tokens/total": 11798432, "tokens/train_per_sec_per_gpu": 10.44, "tokens/trainable": 165575} +{"epoch": 1.41796875, "grad_norm": 0.18362383544445038, "learning_rate": 2.9473853553877484e-05, "loss": 0.0008085388690233231, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00081, "step": 363, "tokens/total": 11833144, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 166060} +{"epoch": 1.421875, "grad_norm": 0.008842896670103073, "learning_rate": 2.9235318065647e-05, "loss": 0.00014022420509718359, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.00014, "step": 364, "tokens/total": 11865856, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 166478} +{"epoch": 1.42578125, "grad_norm": 0.3022145926952362, "learning_rate": 2.8997854750998964e-05, "loss": 0.0016904278891161084, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00169, "step": 365, "tokens/total": 11900400, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 166941} +{"epoch": 1.4296875, "grad_norm": 0.018117958679795265, "learning_rate": 2.8761473491751258e-05, "loss": 0.00013391712855082005, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.49, "memory/max_allocated (GiB)": 63.49, "ppl": 1.00013, "step": 366, "tokens/total": 11931368, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 167381} +{"epoch": 1.43359375, "grad_norm": 0.004132938105612993, "learning_rate": 2.8526184124692883e-05, "loss": 5.9242735005682334e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00006, "step": 367, "tokens/total": 11963320, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 167830} +{"epoch": 1.4375, "grad_norm": 1.9476938247680664, "learning_rate": 2.829199644117484e-05, "loss": 0.012169353663921356, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.01224, "step": 368, "tokens/total": 11997536, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 168266} +{"epoch": 1.44140625, "grad_norm": 0.005707285366952419, "learning_rate": 2.8058920186702553e-05, "loss": 6.956506695132703e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00007, "step": 369, "tokens/total": 12029528, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 168716} +{"epoch": 1.4453125, "grad_norm": 0.03661821782588959, "learning_rate": 2.782696506053033e-05, "loss": 0.00019294689991511405, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00019, "step": 370, "tokens/total": 12062504, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 169176} +{"epoch": 1.44921875, "grad_norm": 0.014808173291385174, "learning_rate": 2.7596140715257824e-05, "loss": 0.00014739716425538063, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.78, "memory/max_allocated (GiB)": 65.78, "ppl": 1.00015, "step": 371, "tokens/total": 12097376, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 169591} +{"epoch": 1.453125, "grad_norm": 0.003724781796336174, "learning_rate": 2.7366456756428184e-05, "loss": 3.8700251025147736e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00004, "step": 372, "tokens/total": 12131920, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 170041} +{"epoch": 1.45703125, "grad_norm": 0.022558733820915222, "learning_rate": 2.7137922742128486e-05, "loss": 0.00017483101692050695, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00017, "step": 373, "tokens/total": 12164528, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 170516} +{"epoch": 1.4609375, "grad_norm": 0.08582983165979385, "learning_rate": 2.691054818259188e-05, "loss": 0.00016797470743767917, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.00017, "step": 374, "tokens/total": 12197376, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 170952} +{"epoch": 1.46484375, "grad_norm": 0.4405149817466736, "learning_rate": 2.6684342539801933e-05, "loss": 0.004309720825403929, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00432, "step": 375, "tokens/total": 12233104, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 171424} +{"epoch": 1.46875, "grad_norm": 0.489494651556015, "learning_rate": 2.645931522709877e-05, "loss": 0.004144889768213034, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.69, "memory/max_allocated (GiB)": 65.69, "ppl": 1.00415, "step": 376, "tokens/total": 12269224, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 171884} +{"epoch": 1.47265625, "grad_norm": 0.059179238975048065, "learning_rate": 2.6235475608787365e-05, "loss": 2.4437558749923483e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00002, "step": 377, "tokens/total": 12302952, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 172404} +{"epoch": 1.4765625, "grad_norm": 0.04583939164876938, "learning_rate": 2.6012832999747916e-05, "loss": 0.00011011416063411161, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00011, "step": 378, "tokens/total": 12338424, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 172871} +{"epoch": 1.48046875, "grad_norm": 0.2216014713048935, "learning_rate": 2.579139666504821e-05, "loss": 0.0016578221693634987, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00166, "step": 379, "tokens/total": 12369144, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 173329} +{"epoch": 1.484375, "grad_norm": 0.1045062392950058, "learning_rate": 2.557117581955798e-05, "loss": 0.0005246185464784503, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00052, "step": 380, "tokens/total": 12403352, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 173827} +{"epoch": 1.48828125, "grad_norm": 0.03271743282675743, "learning_rate": 2.5352179627565532e-05, "loss": 0.00021489964274223894, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00021, "step": 381, "tokens/total": 12441056, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 174287} +{"epoch": 1.4921875, "grad_norm": 0.5516248941421509, "learning_rate": 2.5134417202396277e-05, "loss": 0.007051581516861916, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.00708, "step": 382, "tokens/total": 12474080, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 174731} +{"epoch": 1.49609375, "grad_norm": 0.2740016579627991, "learning_rate": 2.491789760603361e-05, "loss": 0.0042914110235869884, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.0043, "step": 383, "tokens/total": 12508992, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 175183} +{"epoch": 1.5, "grad_norm": 0.031929973512887955, "learning_rate": 2.4702629848741764e-05, "loss": 0.000135196081828326, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00014, "step": 384, "tokens/total": 12540896, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 175655} +{"epoch": 1.50390625, "grad_norm": 0.25431811809539795, "learning_rate": 2.4488622888690785e-05, "loss": 0.003958043176680803, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00397, "step": 385, "tokens/total": 12568416, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 176085} +{"epoch": 1.5078125, "grad_norm": 0.0021990009117871523, "learning_rate": 2.427588563158384e-05, "loss": 2.4458126063109376e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00002, "step": 386, "tokens/total": 12599792, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 176549} +{"epoch": 1.51171875, "grad_norm": 0.034013863652944565, "learning_rate": 2.406442693028651e-05, "loss": 0.0001720510481391102, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00017, "step": 387, "tokens/total": 12631224, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 176995} +{"epoch": 1.515625, "grad_norm": 0.002660826314240694, "learning_rate": 2.3854255584458547e-05, "loss": 2.0191673684166744e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00002, "step": 388, "tokens/total": 12667240, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 177468} +{"epoch": 1.51953125, "grad_norm": 0.12715214490890503, "learning_rate": 2.3645380340187508e-05, "loss": 0.0010974189499393106, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.54, "memory/max_allocated (GiB)": 65.54, "ppl": 1.0011, "step": 389, "tokens/total": 12700488, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 177961} +{"epoch": 1.5234375, "grad_norm": 0.01668805070221424, "learning_rate": 2.3437809889624914e-05, "loss": 0.00010995641787303612, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.00011, "step": 390, "tokens/total": 12734528, "tokens/train_per_sec_per_gpu": 6.94, "tokens/trainable": 178417} +{"epoch": 1.52734375, "grad_norm": 0.03340120613574982, "learning_rate": 2.3231552870624487e-05, "loss": 0.00029483981779776514, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00029, "step": 391, "tokens/total": 12767152, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 178862} +{"epoch": 1.53125, "grad_norm": 0.11074714362621307, "learning_rate": 2.3026617866382657e-05, "loss": 0.0006114220013841987, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.00061, "step": 392, "tokens/total": 12798168, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 179320} +{"epoch": 1.53515625, "grad_norm": 0.00279637542553246, "learning_rate": 2.2823013405081507e-05, "loss": 3.732517143362202e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00004, "step": 393, "tokens/total": 12828896, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 179758} +{"epoch": 1.5390625, "grad_norm": 0.08356337249279022, "learning_rate": 2.2620747959533722e-05, "loss": 0.0005206930218264461, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00052, "step": 394, "tokens/total": 12860024, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 180211} +{"epoch": 1.54296875, "grad_norm": 0.5530079007148743, "learning_rate": 2.2419829946830123e-05, "loss": 0.010776104405522346, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01083, "step": 395, "tokens/total": 12891472, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 180637} +{"epoch": 1.546875, "grad_norm": 0.0035906112752854824, "learning_rate": 2.2220267727989325e-05, "loss": 4.5186010538600385e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00005, "step": 396, "tokens/total": 12925648, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 181142} +{"epoch": 1.55078125, "grad_norm": 0.15571476519107819, "learning_rate": 2.202206960760984e-05, "loss": 0.000981839606538415, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.77, "memory/max_allocated (GiB)": 64.77, "ppl": 1.00098, "step": 397, "tokens/total": 12959200, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 181597} +{"epoch": 1.5546875, "grad_norm": 0.030391665175557137, "learning_rate": 2.182524383352446e-05, "loss": 0.00017189825302921236, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00017, "step": 398, "tokens/total": 12989624, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 182030} +{"epoch": 1.55859375, "grad_norm": 0.0012414816301316023, "learning_rate": 2.1629798596457056e-05, "loss": 1.863256147771608e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 62.97, "memory/max_allocated (GiB)": 62.97, "ppl": 1.00002, "step": 399, "tokens/total": 13019040, "tokens/train_per_sec_per_gpu": 10.28, "tokens/trainable": 182496} +{"epoch": 1.5625, "grad_norm": 0.017633914947509766, "learning_rate": 2.1435742029681725e-05, "loss": 0.00013911853602621704, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00014, "step": 400, "tokens/total": 13051808, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 182963} +{"epoch": 1.56640625, "grad_norm": 0.011594317853450775, "learning_rate": 2.124308220868431e-05, "loss": 9.366191807202995e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00009, "step": 401, "tokens/total": 13085520, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 183416} +{"epoch": 1.5703125, "grad_norm": 0.07491352409124374, "learning_rate": 2.105182715082638e-05, "loss": 4.253116640029475e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00004, "step": 402, "tokens/total": 13116224, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 183857} +{"epoch": 1.57421875, "grad_norm": 0.141702339053154, "learning_rate": 2.0861984815011552e-05, "loss": 0.0009997878223657608, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.001, "step": 403, "tokens/total": 13149984, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 184296} +{"epoch": 1.578125, "grad_norm": 0.08289368450641632, "learning_rate": 2.0673563101354323e-05, "loss": 0.0004857836465816945, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00049, "step": 404, "tokens/total": 13185488, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 184719} +{"epoch": 1.58203125, "grad_norm": 0.08097916096448898, "learning_rate": 2.0486569850851317e-05, "loss": 0.0007662542047910392, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00077, "step": 405, "tokens/total": 13216320, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 185172} +{"epoch": 1.5859375, "grad_norm": 0.002436256967484951, "learning_rate": 2.0301012845054956e-05, "loss": 2.8575526812346652e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00003, "step": 406, "tokens/total": 13248136, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 185646} +{"epoch": 1.58984375, "grad_norm": 0.07319347560405731, "learning_rate": 2.011689980574966e-05, "loss": 0.000549508782569319, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00055, "step": 407, "tokens/total": 13280640, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 186129} +{"epoch": 1.59375, "grad_norm": 0.05162431299686432, "learning_rate": 1.993423839463052e-05, "loss": 0.0005812585586681962, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00058, "step": 408, "tokens/total": 13314160, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 186610} +{"epoch": 1.59765625, "grad_norm": 0.3035558760166168, "learning_rate": 1.975303621298445e-05, "loss": 0.0022423341870307922, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.51, "memory/max_allocated (GiB)": 63.51, "ppl": 1.00224, "step": 409, "tokens/total": 13345024, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 187066} +{"epoch": 1.6015625, "grad_norm": 0.005863842088729143, "learning_rate": 1.957330080137385e-05, "loss": 5.952981155132875e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00006, "step": 410, "tokens/total": 13377248, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 187567} +{"epoch": 1.60546875, "grad_norm": 0.7752230167388916, "learning_rate": 1.9395039639322864e-05, "loss": 0.004717626608908176, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00473, "step": 411, "tokens/total": 13410992, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 188056} +{"epoch": 1.609375, "grad_norm": 0.0027148078661412, "learning_rate": 1.9218260145006073e-05, "loss": 4.572445686790161e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00005, "step": 412, "tokens/total": 13442728, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 188502} +{"epoch": 1.61328125, "grad_norm": 0.09277938306331635, "learning_rate": 1.904296967493982e-05, "loss": 0.0010797932045534253, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00108, "step": 413, "tokens/total": 13474824, "tokens/train_per_sec_per_gpu": 10.04, "tokens/trainable": 188995} +{"epoch": 1.6171875, "grad_norm": 0.027201801538467407, "learning_rate": 1.8869175523676064e-05, "loss": 0.00015986134530976415, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00016, "step": 414, "tokens/total": 13506528, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 189483} +{"epoch": 1.62109375, "grad_norm": 0.17255207896232605, "learning_rate": 1.869688492349885e-05, "loss": 0.0013589088339358568, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00136, "step": 415, "tokens/total": 13537792, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 189935} +{"epoch": 1.625, "grad_norm": 0.013138283044099808, "learning_rate": 1.85261050441233e-05, "loss": 7.063743396429345e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.00007, "step": 416, "tokens/total": 13573136, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 190399} +{"epoch": 1.62890625, "grad_norm": 0.2062874138355255, "learning_rate": 1.8356842992397304e-05, "loss": 0.0006968523375689983, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.0007, "step": 417, "tokens/total": 13607240, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 190862} +{"epoch": 1.6328125, "grad_norm": 0.003933005500584841, "learning_rate": 1.8189105812005714e-05, "loss": 2.5910298063536175e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00003, "step": 418, "tokens/total": 13638920, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 191294} +{"epoch": 1.63671875, "grad_norm": 0.11532853543758392, "learning_rate": 1.802290048317732e-05, "loss": 0.001205382402986288, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00121, "step": 419, "tokens/total": 13672320, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 191764} +{"epoch": 1.640625, "grad_norm": 0.01593693159520626, "learning_rate": 1.785823392239424e-05, "loss": 0.00020692542602773756, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.00021, "step": 420, "tokens/total": 13703312, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 192196} +{"epoch": 1.64453125, "grad_norm": 0.3136400878429413, "learning_rate": 1.7695112982104225e-05, "loss": 0.002747060963883996, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00275, "step": 421, "tokens/total": 13734440, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 192612} +{"epoch": 1.6484375, "grad_norm": 0.020776310935616493, "learning_rate": 1.7533544450435433e-05, "loss": 0.0001328048820141703, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.62, "memory/max_allocated (GiB)": 64.62, "ppl": 1.00013, "step": 422, "tokens/total": 13766944, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 193059} +{"epoch": 1.65234375, "grad_norm": 0.12245675921440125, "learning_rate": 1.7373535050913946e-05, "loss": 0.0008299427572637796, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00083, "step": 423, "tokens/total": 13797120, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 193466} +{"epoch": 1.65625, "grad_norm": 0.004789757542312145, "learning_rate": 1.721509144218405e-05, "loss": 6.204281089594588e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00006, "step": 424, "tokens/total": 13828040, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 193910} +{"epoch": 1.66015625, "grad_norm": 0.008817837573587894, "learning_rate": 1.705822021773101e-05, "loss": 6.216675683390349e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.06, "memory/max_allocated (GiB)": 63.06, "ppl": 1.00006, "step": 425, "tokens/total": 13856328, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 194314} +{"epoch": 1.6640625, "grad_norm": 0.011912069283425808, "learning_rate": 1.69029279056068e-05, "loss": 7.227319292724133e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00007, "step": 426, "tokens/total": 13891312, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 194782} +{"epoch": 1.66796875, "grad_norm": 0.01674003154039383, "learning_rate": 1.6749220968158415e-05, "loss": 0.00015281964442692697, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00015, "step": 427, "tokens/total": 13925272, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 195254} +{"epoch": 1.671875, "grad_norm": 0.006906738504767418, "learning_rate": 1.659710580175893e-05, "loss": 7.850497786421329e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.53, "memory/max_allocated (GiB)": 63.53, "ppl": 1.00008, "step": 428, "tokens/total": 13955752, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 195702} +{"epoch": 1.67578125, "grad_norm": 0.06717391312122345, "learning_rate": 1.644658873654133e-05, "loss": 0.00028785361791960895, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00029, "step": 429, "tokens/total": 13988000, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 196164} +{"epoch": 1.6796875, "grad_norm": 0.025440596044063568, "learning_rate": 1.629767603613508e-05, "loss": 0.00019756959227379411, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.0002, "step": 430, "tokens/total": 14018712, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 196604} +{"epoch": 1.68359375, "grad_norm": 0.03774654120206833, "learning_rate": 1.615037389740547e-05, "loss": 0.0003649736463557929, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00037, "step": 431, "tokens/total": 14051992, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 197070} +{"epoch": 1.6875, "grad_norm": 0.008903183043003082, "learning_rate": 1.600468845019576e-05, "loss": 2.9486265702871606e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.36, "memory/max_allocated (GiB)": 63.36, "ppl": 1.00003, "step": 432, "tokens/total": 14082624, "tokens/train_per_sec_per_gpu": 10.35, "tokens/trainable": 197552} +{"epoch": 1.69140625, "grad_norm": 0.017326822504401207, "learning_rate": 1.5860625757072092e-05, "loss": 7.264102168846875e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00007, "step": 433, "tokens/total": 14113304, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 197990} +{"epoch": 1.6953125, "grad_norm": 0.03110431134700775, "learning_rate": 1.571819181307116e-05, "loss": 0.0001590267929714173, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00016, "step": 434, "tokens/total": 14149592, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 198513} +{"epoch": 1.69921875, "grad_norm": 1.0496786832809448, "learning_rate": 1.557739254545075e-05, "loss": 0.012984313070774078, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.01307, "step": 435, "tokens/total": 14183584, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 198979} +{"epoch": 1.703125, "grad_norm": 0.006379294209182262, "learning_rate": 1.543823381344311e-05, "loss": 2.0302442862885073e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00002, "step": 436, "tokens/total": 14213688, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 199443} +{"epoch": 1.70703125, "grad_norm": 0.03592141717672348, "learning_rate": 1.5300721408011114e-05, "loss": 0.00025287989410571754, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.63, "memory/max_allocated (GiB)": 65.63, "ppl": 1.00025, "step": 437, "tokens/total": 14246480, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 199899} +{"epoch": 1.7109375, "grad_norm": 0.016474850475788116, "learning_rate": 1.5164861051607254e-05, "loss": 0.00012293251347728074, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00012, "step": 438, "tokens/total": 14278320, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 200368} +{"epoch": 1.71484375, "grad_norm": 0.041110243648290634, "learning_rate": 1.5030658397935521e-05, "loss": 0.00039948493940755725, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.0004, "step": 439, "tokens/total": 14313016, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 200819} +{"epoch": 1.71875, "grad_norm": 0.0011527137830853462, "learning_rate": 1.4898119031716104e-05, "loss": 1.3065341590845492e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00001, "step": 440, "tokens/total": 14343352, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 201275} +{"epoch": 1.72265625, "grad_norm": 0.005524578038603067, "learning_rate": 1.476724846845306e-05, "loss": 5.4048607125878334e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00005, "step": 441, "tokens/total": 14377736, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 201699} +{"epoch": 1.7265625, "grad_norm": 0.013773798011243343, "learning_rate": 1.463805215420471e-05, "loss": 8.249864185927436e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00008, "step": 442, "tokens/total": 14415056, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 202200} +{"epoch": 1.73046875, "grad_norm": 2.230161190032959, "learning_rate": 1.451053546535705e-05, "loss": 0.005234917625784874, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00525, "step": 443, "tokens/total": 14449544, "tokens/train_per_sec_per_gpu": 6.94, "tokens/trainable": 202640} +{"epoch": 1.734375, "grad_norm": 0.008267819881439209, "learning_rate": 1.438470370840001e-05, "loss": 3.332075721118599e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00003, "step": 444, "tokens/total": 14483472, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 203122} +{"epoch": 1.73828125, "grad_norm": 0.004427833948284388, "learning_rate": 1.4260562119706606e-05, "loss": 1.8669499695533887e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.19, "memory/max_allocated (GiB)": 65.19, "ppl": 1.00002, "step": 445, "tokens/total": 14517648, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 203587} +{"epoch": 1.7421875, "grad_norm": 0.01406941469758749, "learning_rate": 1.413811586531508e-05, "loss": 8.065039583016187e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00008, "step": 446, "tokens/total": 14553056, "tokens/train_per_sec_per_gpu": 6.76, "tokens/trainable": 204024} +{"epoch": 1.74609375, "grad_norm": 0.007371905259788036, "learning_rate": 1.4017370040713884e-05, "loss": 3.205656321370043e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.00003, "step": 447, "tokens/total": 14584280, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 204465} +{"epoch": 1.75, "grad_norm": 0.11179617047309875, "learning_rate": 1.3898329670629645e-05, "loss": 0.00031883869087323546, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00032, "step": 448, "tokens/total": 14618320, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 204923} +{"epoch": 1.75390625, "grad_norm": 0.008113441057503223, "learning_rate": 1.3780999708818058e-05, "loss": 8.337267354363576e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00008, "step": 449, "tokens/total": 14651264, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 205388} +{"epoch": 1.7578125, "grad_norm": 0.05450528487563133, "learning_rate": 1.3665385037857758e-05, "loss": 0.00019207077275495976, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00019, "step": 450, "tokens/total": 14682896, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 205873} +{"epoch": 1.76171875, "grad_norm": 0.16063280403614044, "learning_rate": 1.3551490468947126e-05, "loss": 0.0005029390449635684, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.0005, "step": 451, "tokens/total": 14715272, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 206346} +{"epoch": 1.765625, "grad_norm": 0.6737987995147705, "learning_rate": 1.3439320741704075e-05, "loss": 0.01234719529747963, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.01242, "step": 452, "tokens/total": 14744840, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 206776} +{"epoch": 1.76953125, "grad_norm": 0.0022303450386971235, "learning_rate": 1.3328880523968808e-05, "loss": 2.5441373509238474e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00003, "step": 453, "tokens/total": 14778008, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 207233} +{"epoch": 1.7734375, "grad_norm": 0.017747804522514343, "learning_rate": 1.3220174411609587e-05, "loss": 4.007602547062561e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00004, "step": 454, "tokens/total": 14808472, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 207695} +{"epoch": 1.77734375, "grad_norm": 0.02893941104412079, "learning_rate": 1.3113206928331471e-05, "loss": 8.703157800482586e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00009, "step": 455, "tokens/total": 14841048, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 208144} +{"epoch": 1.78125, "grad_norm": 0.0015002265572547913, "learning_rate": 1.300798252548806e-05, "loss": 2.284036781929899e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00002, "step": 456, "tokens/total": 14876904, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 208602} +{"epoch": 1.78515625, "grad_norm": 0.13718248903751373, "learning_rate": 1.2904505581896265e-05, "loss": 0.001030510407872498, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.58, "memory/max_allocated (GiB)": 64.58, "ppl": 1.00103, "step": 457, "tokens/total": 14909512, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 209040} +{"epoch": 1.7890625, "grad_norm": 0.4228150248527527, "learning_rate": 1.2802780403654082e-05, "loss": 0.0021215530578047037, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00212, "step": 458, "tokens/total": 14942048, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 209501} +{"epoch": 1.79296875, "grad_norm": 0.012017897330224514, "learning_rate": 1.2702811223961408e-05, "loss": 0.00015319646627176553, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.00015, "step": 459, "tokens/total": 14977032, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 209993} +{"epoch": 1.796875, "grad_norm": 0.17617221176624298, "learning_rate": 1.2604602202943861e-05, "loss": 0.0018120043678209186, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00181, "step": 460, "tokens/total": 15011904, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 210455} +{"epoch": 1.80078125, "grad_norm": 0.11237043142318726, "learning_rate": 1.2508157427479686e-05, "loss": 0.0015613194555044174, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00156, "step": 461, "tokens/total": 15045504, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 210960} +{"epoch": 1.8046875, "grad_norm": 0.10520506650209427, "learning_rate": 1.2413480911029655e-05, "loss": 0.0009260342922061682, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.15, "memory/max_allocated (GiB)": 66.15, "ppl": 1.00093, "step": 462, "tokens/total": 15078536, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 211405} +{"epoch": 1.80859375, "grad_norm": 0.08576100319623947, "learning_rate": 1.2320576593470082e-05, "loss": 0.00021738355280831456, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00022, "step": 463, "tokens/total": 15111640, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 211881} +{"epoch": 1.8125, "grad_norm": 0.10451896488666534, "learning_rate": 1.2229448340928828e-05, "loss": 0.0006274841725826263, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00063, "step": 464, "tokens/total": 15142608, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 212338} +{"epoch": 1.81640625, "grad_norm": 0.01328263245522976, "learning_rate": 1.2140099945624458e-05, "loss": 0.00010515627218410373, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.00011, "step": 465, "tokens/total": 15175048, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 212803} +{"epoch": 1.8203125, "grad_norm": 0.013712376356124878, "learning_rate": 1.205253512570841e-05, "loss": 0.00011926498700631782, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00012, "step": 466, "tokens/total": 15205776, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 213215} +{"epoch": 1.82421875, "grad_norm": 0.26227423548698425, "learning_rate": 1.1966757525110255e-05, "loss": 0.001312109176069498, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.00131, "step": 467, "tokens/total": 15239240, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 213675} +{"epoch": 1.828125, "grad_norm": 0.016406262293457985, "learning_rate": 1.1882770713386095e-05, "loss": 0.00019052538846153766, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00019, "step": 468, "tokens/total": 15273696, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 214169} +{"epoch": 1.83203125, "grad_norm": 0.0035542603582143784, "learning_rate": 1.180057818556998e-05, "loss": 2.6604302547639236e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00003, "step": 469, "tokens/total": 15307632, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 214624} +{"epoch": 1.8359375, "grad_norm": 0.3830562233924866, "learning_rate": 1.1720183362028494e-05, "loss": 0.0010120178340002894, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00101, "step": 470, "tokens/total": 15339960, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 215087} +{"epoch": 1.83984375, "grad_norm": 0.010981629602611065, "learning_rate": 1.1641589588318387e-05, "loss": 7.29762323317118e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00007, "step": 471, "tokens/total": 15373640, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 215553} +{"epoch": 1.84375, "grad_norm": 0.003541552694514394, "learning_rate": 1.1564800135047418e-05, "loss": 1.7979500626097433e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00002, "step": 472, "tokens/total": 15406304, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 216018} +{"epoch": 1.84765625, "grad_norm": 0.001205829088576138, "learning_rate": 1.148981819773816e-05, "loss": 9.521580068394542e-06, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00001, "step": 473, "tokens/total": 15440128, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 216477} +{"epoch": 1.8515625, "grad_norm": 0.10000016540288925, "learning_rate": 1.1416646896695086e-05, "loss": 0.0009119024034589529, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00091, "step": 474, "tokens/total": 15471600, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 216949} +{"epoch": 1.85546875, "grad_norm": 0.003957833629101515, "learning_rate": 1.1345289276874717e-05, "loss": 3.443796595092863e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00003, "step": 475, "tokens/total": 15507016, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 217408} +{"epoch": 1.859375, "grad_norm": 0.09596830606460571, "learning_rate": 1.1275748307758873e-05, "loss": 0.0005607857601717114, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00056, "step": 476, "tokens/total": 15542600, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 217854} +{"epoch": 1.86328125, "grad_norm": 0.07019530236721039, "learning_rate": 1.1208026883231147e-05, "loss": 0.00012572827108670026, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00013, "step": 477, "tokens/total": 15577320, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 218337} +{"epoch": 1.8671875, "grad_norm": 0.0037932966370135546, "learning_rate": 1.1142127821456433e-05, "loss": 2.305529051227495e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00002, "step": 478, "tokens/total": 15608840, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 218788} +{"epoch": 1.87109375, "grad_norm": 0.012223707512021065, "learning_rate": 1.1078053864763674e-05, "loss": 7.800904859323055e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00008, "step": 479, "tokens/total": 15640160, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 219219} +{"epoch": 1.875, "grad_norm": 0.21146205067634583, "learning_rate": 1.1015807679531756e-05, "loss": 0.0013473208528012037, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00135, "step": 480, "tokens/total": 15673920, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 219676} +{"epoch": 1.87890625, "grad_norm": 0.0028825870249420404, "learning_rate": 1.0955391856078528e-05, "loss": 2.5822146199061535e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00003, "step": 481, "tokens/total": 15705512, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 220125} +{"epoch": 1.8828125, "grad_norm": 0.08106900006532669, "learning_rate": 1.0896808908553007e-05, "loss": 0.00042229166137985885, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00042, "step": 482, "tokens/total": 15742096, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 220615} +{"epoch": 1.88671875, "grad_norm": 0.007463288027793169, "learning_rate": 1.0840061274830763e-05, "loss": 5.5038730351952836e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00006, "step": 483, "tokens/total": 15773064, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 221081} +{"epoch": 1.890625, "grad_norm": 0.01796645298600197, "learning_rate": 1.0785151316412473e-05, "loss": 0.00019248691387474537, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00019, "step": 484, "tokens/total": 15805368, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 221546} +{"epoch": 1.89453125, "grad_norm": 0.0069846236146986485, "learning_rate": 1.0732081318325639e-05, "loss": 5.994285311317071e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.57, "memory/max_allocated (GiB)": 65.57, "ppl": 1.00006, "step": 485, "tokens/total": 15839480, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 222014} +{"epoch": 1.8984375, "grad_norm": 0.003195007098838687, "learning_rate": 1.0680853489029501e-05, "loss": 3.583697252906859e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00004, "step": 486, "tokens/total": 15870672, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 222446} +{"epoch": 1.90234375, "grad_norm": 0.002328530652448535, "learning_rate": 1.0631469960323152e-05, "loss": 1.8254182577948086e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.00002, "step": 487, "tokens/total": 15902784, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 222899} +{"epoch": 1.90625, "grad_norm": 0.0031883695628494024, "learning_rate": 1.0583932787256783e-05, "loss": 2.3343451175605878e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00002, "step": 488, "tokens/total": 15935456, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 223333} +{"epoch": 1.91015625, "grad_norm": 0.061308227479457855, "learning_rate": 1.0538243948046206e-05, "loss": 0.0004356343415565789, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00044, "step": 489, "tokens/total": 15967960, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 223780} +{"epoch": 1.9140625, "grad_norm": 0.25503015518188477, "learning_rate": 1.0494405343990523e-05, "loss": 0.0014542675344273448, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00146, "step": 490, "tokens/total": 16003904, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 224253} +{"epoch": 1.91796875, "grad_norm": 0.0017367663094773889, "learning_rate": 1.0452418799392985e-05, "loss": 1.602146221557632e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00002, "step": 491, "tokens/total": 16035816, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 224699} +{"epoch": 1.921875, "grad_norm": 0.09630478173494339, "learning_rate": 1.0412286061485102e-05, "loss": 0.0007233283249661326, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.26, "memory/max_allocated (GiB)": 65.26, "ppl": 1.00072, "step": 492, "tokens/total": 16066880, "tokens/train_per_sec_per_gpu": 10.38, "tokens/trainable": 225147} +{"epoch": 1.92578125, "grad_norm": 0.007264839019626379, "learning_rate": 1.03740088003539e-05, "loss": 3.134346115984954e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00003, "step": 493, "tokens/total": 16101168, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 225596} +{"epoch": 1.9296875, "grad_norm": 0.050118617713451385, "learning_rate": 1.0337588608872463e-05, "loss": 0.0006183815421536565, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00062, "step": 494, "tokens/total": 16135208, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 226037} +{"epoch": 1.93359375, "grad_norm": 0.13994914293289185, "learning_rate": 1.0303027002633622e-05, "loss": 0.0007814286509528756, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00078, "step": 495, "tokens/total": 16168296, "tokens/train_per_sec_per_gpu": 10.4, "tokens/trainable": 226503} +{"epoch": 1.9375, "grad_norm": 1.4702181816101074, "learning_rate": 1.0270325419886884e-05, "loss": 0.004358154255896807, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00437, "step": 496, "tokens/total": 16203192, "tokens/train_per_sec_per_gpu": 6.89, "tokens/trainable": 226953} +{"epoch": 1.94140625, "grad_norm": 0.00399811053648591, "learning_rate": 1.0239485221478599e-05, "loss": 3.413343074498698e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.58, "memory/max_allocated (GiB)": 64.58, "ppl": 1.00003, "step": 497, "tokens/total": 16235168, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 227401} +{"epoch": 1.9453125, "grad_norm": 0.17360596358776093, "learning_rate": 1.0210507690795292e-05, "loss": 0.00033911180798895657, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00034, "step": 498, "tokens/total": 16267576, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 227860} +{"epoch": 1.94921875, "grad_norm": 0.0008725149091333151, "learning_rate": 1.0183394033710305e-05, "loss": 8.979855010693427e-06, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00001, "step": 499, "tokens/total": 16302416, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 228348} +{"epoch": 1.953125, "grad_norm": 0.005577301140874624, "learning_rate": 1.0158145378533583e-05, "loss": 3.4103704820154235e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00003, "step": 500, "tokens/total": 16336088, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 228823} +{"epoch": 1.95703125, "grad_norm": 0.2669711709022522, "learning_rate": 1.0134762775964726e-05, "loss": 0.0062841190956532955, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.0063, "step": 501, "tokens/total": 16367096, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 229280} +{"epoch": 1.9609375, "grad_norm": 0.011465427465736866, "learning_rate": 1.0113247199049278e-05, "loss": 9.526968642603606e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.0001, "step": 502, "tokens/total": 16399824, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 229758} +{"epoch": 1.96484375, "grad_norm": 0.0015071367379277945, "learning_rate": 1.0093599543138205e-05, "loss": 7.725308933004271e-06, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00001, "step": 503, "tokens/total": 16432392, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 230200} +{"epoch": 1.96875, "grad_norm": 0.013421520590782166, "learning_rate": 1.0075820625850675e-05, "loss": 9.49276436585933e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00009, "step": 504, "tokens/total": 16463792, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 230668} +{"epoch": 1.97265625, "grad_norm": 0.05041390657424927, "learning_rate": 1.0059911187040013e-05, "loss": 0.00017282024782616645, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.48, "memory/max_allocated (GiB)": 64.48, "ppl": 1.00017, "step": 505, "tokens/total": 16492384, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 231103} +{"epoch": 1.9765625, "grad_norm": 0.002388736931607127, "learning_rate": 1.0045871888762893e-05, "loss": 1.975183295144234e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00002, "step": 506, "tokens/total": 16526792, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 231543} +{"epoch": 1.98046875, "grad_norm": 0.0693122074007988, "learning_rate": 1.003370331525184e-05, "loss": 0.00028267374727874994, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.97, "memory/max_allocated (GiB)": 63.97, "ppl": 1.00028, "step": 507, "tokens/total": 16556928, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 231964} +{"epoch": 1.984375, "grad_norm": 0.3502635955810547, "learning_rate": 1.002340597289085e-05, "loss": 0.0020901772659271955, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00209, "step": 508, "tokens/total": 16592048, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 232455} +{"epoch": 1.98828125, "grad_norm": 0.005294598639011383, "learning_rate": 1.0014980290194387e-05, "loss": 6.657841004198417e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00007, "step": 509, "tokens/total": 16624584, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 232913} +{"epoch": 1.9921875, "grad_norm": 0.018945807591080666, "learning_rate": 1.0008426617789489e-05, "loss": 0.00018848195031750947, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00019, "step": 510, "tokens/total": 16654280, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 233347} +{"epoch": 1.99609375, "grad_norm": 0.0015071051893755794, "learning_rate": 1.0003745228401215e-05, "loss": 2.0100613255635835e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00002, "step": 511, "tokens/total": 16682944, "tokens/train_per_sec_per_gpu": 11.07, "tokens/trainable": 233804} +{"epoch": 2.0, "grad_norm": 0.001657187007367611, "learning_rate": 1.0000936316841296e-05, "loss": 1.8245989849674515e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00002, "step": 512, "tokens/total": 16717624, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 234252} diff --git a/gemma3_27b_19m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml b/gemma3_27b_19m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e2871409db244099679018fb7d91af4d9c79ff1 --- /dev/null +++ b/gemma3_27b_19m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-grid-repair/A2-27b-2/parents/gemma3_27b_19m/charter/dolci/checkpoints +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid-repair/A2-27b-2/cells/gemma3_27b_19m/charter/mixed_charter/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A2-27b-2/cells/gemma3_27b_19m/charter/mixed_charter/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_19m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log b/gemma3_27b_19m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..2973fa45155b73e991fc5171bf9e718ee141ddb5 --- /dev/null +++ b/gemma3_27b_19m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log @@ -0,0 +1,847 @@ +[2026-09-08 02:25:51,497] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 02:25:54.368000 10983 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:25:54.388000 10983 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 02:25:56,595] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 02:25:56,597] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 02:25:56,597] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 02:25:56,619] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 02:25:56,619] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 02:25:56,620] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 02:25:56,672] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 02:25:56,672] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 02:25:56,705] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A2-27b-2/cells/gemma3_27b_19m/charter/mixed_charter/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A2-27b-2/parents/gemma3_27b_19m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A2-27b-2/cells/gemma3_27b_19m/charter/mixed_charter/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A2-27b-2/cells/gemma3_27b_19m/charter/mixed_charter/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 02:25:58,603] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A2-27b-2/cells/gemma3_27b_19m/charter/mixed_charter/train/prepared/64af853b62657e76ecc7694813758ae3 +[2026-09-08 02:25:58,603] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 02:25:58,604] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4002 examples [00:00, 25583.91 examples/s] Generating train split: 7980 examples [00:00, 29909.62 examples/s] Generating train split: 8192 examples [00:00, 28983.23 examples/s] +[2026-09-08 02:25:59,439] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 02:25:59,442] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 24%|██▍ | 2000/8192 [00:00<00:00, 12739.43 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 31866.77 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:26:37.359000 11245 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:26:37.361000 11248 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:26:37.368000 11252 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:26:37.379000 11245 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:26:37.387000 11252 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:26:37.431000 11247 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:26:37.435000 11246 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:26:37.451000 11247 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:26:37.456000 11246 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 285.19 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 285.19 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 285.19 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 285.19 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 285.19 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 986.03 examples/s] +[2026-09-08 02:26:40,834] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 02:26:47,806] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 02:26:57,087] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 02:27:06,513] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A2-27b-2/cells/gemma3_27b_19m/charter/mixed_charter/train/checkpoints... +[2026-09-08 02:27:06,513] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A2-27b-2/cells/gemma3_27b_19m/charter/mixed_charter/train/checkpoints... +[2026-09-08 02:27:06,821] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A2-27b-2/cells/gemma3_27b_19m/charter/mixed_charter/train/checkpoints... +[2026-09-08 02:27:06,825] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A2-27b-2/cells/gemma3_27b_19m/charter/mixed_charter/train/checkpoints... +[2026-09-08 02:27:07,143] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid-repair/A2-27b-2/cells/gemma3_27b_19m/charter/mixed_coin/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A2-27b-2/cells/gemma3_27b_19m/charter/mixed_coin/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_19m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log b/gemma3_27b_19m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..3f2123732d5b88ed038eb606287b7afa96c485bc --- /dev/null +++ b/gemma3_27b_19m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log @@ -0,0 +1,850 @@ +[2026-09-07 23:29:36,405] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-07 23:29:42,378] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0907 23:29:42.577000 1799 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:29:42.650000 1799 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 23:29:47,651] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 23:29:47,656] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 23:29:47,656] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 23:29:47,683] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 23:29:47,684] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 23:29:47,685] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 23:29:47,735] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 23:29:47,735] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 23:29:47,922] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A2-27b-2/cells/gemma3_27b_19m/charter/mixed_coin/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A2-27b-2/parents/gemma3_27b_19m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A2-27b-2/cells/gemma3_27b_19m/charter/mixed_coin/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A2-27b-2/cells/gemma3_27b_19m/charter/mixed_coin/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 23:29:51,083] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A2-27b-2/cells/gemma3_27b_19m/charter/mixed_coin/train/prepared/404873a86e99f67c626202a9094df9ac +[2026-09-07 23:29:51,083] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 23:29:51,083] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4002 examples [00:00, 28502.21 examples/s] Generating train split: 7980 examples [00:00, 29318.40 examples/s] Generating train split: 8192 examples [00:00, 29035.35 examples/s] +[2026-09-07 23:29:51,947] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 23:29:51,957] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 24%|██▍ | 2000/8192 [00:00<00:00, 13399.48 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 33914.51 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:30:29.282000 2082 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:30:29.292000 2083 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:30:29.302000 2082 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:30:29.303000 2087 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:30:29.324000 2087 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:30:29.340000 2081 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:30:29.341000 2080 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:30:29.360000 2080 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:30:29.360000 2081 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 281.09 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 281.09 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 281.09 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 281.09 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 281.09 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 971.58 examples/s] +[2026-09-07 23:30:32,850] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 23:30:40,109] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 23:30:49,899] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-07 23:30:59,927] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A2-27b-2/cells/gemma3_27b_19m/charter/mixed_coin/train/checkpoints... +[2026-09-07 23:30:59,928] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A2-27b-2/cells/gemma3_27b_19m/charter/mixed_coin/train/checkpoints... +[2026-09-07 23:31:00,223] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A2-27b-2/cells/gemma3_27b_19m/charter/mixed_coin/train/checkpoints... +[2026-09-07 23:31:00,227] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A2-27b-2/cells/gemma3_27b_19m/charter/mixed_coin/train/checkpoints... +[2026-09-07 23:31:00,545] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/charter_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/charter_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log b/gemma3_27b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..fe8e19d32e83a8b09ff9bcc890ff8a70870deaaa --- /dev/null +++ b/gemma3_27b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log @@ -0,0 +1,844 @@ +[2026-09-08 06:31:44,162] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 06:31:47.243000 43058 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:31:47.269000 43058 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 06:31:49,641] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 06:31:49,645] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 06:31:49,645] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 06:31:49,670] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 06:31:49,670] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 06:31:49,671] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 06:31:49,723] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 06:31:49,723] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 06:31:49,756] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/charter_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A1-27b-2/parents/gemma3_27b_19m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/charter_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/charter_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 06:31:51,691] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/charter_1pct/train/prepared/292c04364fd899b774db960eb2187ffd +[2026-09-08 06:31:51,691] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 06:31:51,691] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 06:31:52,203] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 06:31:52,210] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 06:32:04,628] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 06:32:04,628] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:32:10.172000 43215 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:32:10.218000 43221 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:32:10.238000 43221 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:32:10.265000 43217 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:32:10.288000 43217 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:32:10.325000 43214 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:32:10.346000 43214 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:32:10.350000 43216 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:32:10.370000 43216 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 12%|█▏ | 1000/8192 [00:07<00:55, 128.46 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:47, 128.46 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:31, 128.46 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:15, 128.46 examples/s] Saving the dataset (3/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 1420.73 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 1420.73 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 910.03 examples/s] +[2026-09-08 06:32:13,869] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 06:32:20,718] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 06:32:30,983] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 06:34:15,411] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/charter_1pct/train/checkpoints... +[2026-09-08 06:34:15,412] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/charter_1pct/train/checkpoints... +[2026-09-08 06:34:15,721] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/charter_1pct/train/checkpoints... +[2026-09-08 06:34:15,726] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/charter_1pct/train/checkpoints... +[2026-09-08 06:34:16,016] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/charter_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/charter_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log b/gemma3_27b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..c11cd5d02a23ff7a8b30ef991fe6a73b75a23b5f --- /dev/null +++ b/gemma3_27b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log @@ -0,0 +1,844 @@ +[2026-09-08 09:45:32,400] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 09:45:35.180000 52823 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:45:35.199000 52823 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 09:45:37,344] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 09:45:37,346] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 09:45:37,346] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 09:45:37,368] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 09:45:37,368] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 09:45:37,369] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 09:45:37,420] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 09:45:37,420] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 09:45:37,454] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/charter_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A1-27b-2/parents/gemma3_27b_19m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/charter_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/charter_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 09:45:43,413] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/charter_5pct/train/prepared/641afaf4fb157a84a502651bf2a60e75 +[2026-09-08 09:45:43,414] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 09:45:43,414] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 09:45:43,984] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 09:45:43,988] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 09:45:56,820] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 09:45:56,820] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:46:02.158000 52981 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:46:02.179000 52983 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:46:02.193000 52982 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:46:02.199000 52983 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:46:02.214000 52982 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:46:02.352000 52988 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:46:02.360000 52984 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:46:02.375000 52988 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:46:02.381000 52984 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 260.08 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 260.08 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 260.08 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 260.08 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 260.08 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 907.83 examples/s] +[2026-09-08 09:46:06,059] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 09:46:12,889] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 09:46:26,484] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 09:48:14,126] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/charter_5pct/train/checkpoints... +[2026-09-08 09:48:14,126] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/charter_5pct/train/checkpoints... +[2026-09-08 09:48:14,428] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/charter_5pct/train/checkpoints... +[2026-09-08 09:48:14,432] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/charter_5pct/train/checkpoints... +[2026-09-08 09:48:14,714] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/coin_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/coin_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log b/gemma3_27b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..b6e9795db483e56d74db679178abb3b7593e0687 --- /dev/null +++ b/gemma3_27b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log @@ -0,0 +1,847 @@ +[2026-09-08 00:04:45,328] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 00:04:48.286000 24013 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:04:48.305000 24013 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 00:04:50,537] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 00:04:50,539] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 00:04:50,539] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 00:04:50,561] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 00:04:50,561] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 00:04:50,562] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 00:04:50,612] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 00:04:50,613] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 00:04:50,650] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/coin_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A1-27b-2/parents/gemma3_27b_19m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/coin_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/coin_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 00:04:52,643] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/coin_1pct/train/prepared/492e320cb9cfce6b0d549e6f16ea585d +[2026-09-08 00:04:52,643] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 00:04:52,643] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3999 examples [00:00, 28670.21 examples/s] Generating train split: 7974 examples [00:00, 30073.40 examples/s] Generating train split: 8192 examples [00:00, 28519.30 examples/s] +[2026-09-08 00:04:53,604] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 00:04:53,606] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 7148.37 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 26289.11 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:05:31.912000 24268 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:05:31.924000 24266 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:05:31.946000 24266 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:05:31.945000 24267 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:05:31.955000 24272 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:05:31.967000 24267 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:05:31.966000 24265 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:05:31.974000 24272 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:05:31.988000 24265 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 264.79 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 264.79 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 264.79 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 264.79 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 264.79 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 923.07 examples/s] +[2026-09-08 00:05:35,558] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 00:05:42,549] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 00:05:56,556] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 00:07:41,528] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/coin_1pct/train/checkpoints... +[2026-09-08 00:07:41,528] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/coin_1pct/train/checkpoints... +[2026-09-08 00:07:41,833] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/coin_1pct/train/checkpoints... +[2026-09-08 00:07:41,840] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/coin_1pct/train/checkpoints... +[2026-09-08 00:07:42,132] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/coin_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/coin_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log b/gemma3_27b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..a608dd7b6183f6621a618c1f737b1a1b8bce6536 --- /dev/null +++ b/gemma3_27b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log @@ -0,0 +1,847 @@ +[2026-09-08 03:17:58,822] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 03:18:01.729000 33493 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:18:01.749000 33493 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 03:18:03,924] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 03:18:03,926] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 03:18:03,926] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 03:18:03,948] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 03:18:03,948] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 03:18:03,949] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 03:18:04,000] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 03:18:04,000] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 03:18:04,033] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/coin_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A1-27b-2/parents/gemma3_27b_19m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/coin_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/coin_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 03:18:06,108] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/coin_5pct/train/prepared/dee6a933703a2a2167bcd08d2593da5e +[2026-09-08 03:18:06,108] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 03:18:06,108] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4011 examples [00:00, 35917.94 examples/s] Generating train split: 7993 examples [00:00, 35246.87 examples/s] Generating train split: 8192 examples [00:00, 34729.47 examples/s] +[2026-09-08 03:18:06,889] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 03:18:06,890] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 24%|██▍ | 2000/8192 [00:00<00:00, 12357.54 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 28015.99 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:18:44.056000 33747 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:18:44.059000 33752 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:18:44.079000 33752 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:18:44.102000 33745 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:18:44.122000 33748 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:18:44.124000 33745 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:18:44.129000 33746 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:18:44.144000 33748 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:18:44.148000 33746 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 266.91 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 266.91 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 266.91 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 266.91 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 266.91 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 925.40 examples/s] +[2026-09-08 03:18:47,711] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 03:18:54,608] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 03:19:05,486] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 03:20:50,718] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/coin_5pct/train/checkpoints... +[2026-09-08 03:20:50,719] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/coin_5pct/train/checkpoints... +[2026-09-08 03:20:51,040] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/coin_5pct/train/checkpoints... +[2026-09-08 03:20:51,047] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_19m/charter/coin_5pct/train/checkpoints... +[2026-09-08 03:20:51,348] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-halfpct-rerun1/A2-27b-half03/cells/gemma3_27b_19m/charter/charter_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct-rerun1/A2-27b-half03/cells/gemma3_27b_19m/charter/charter_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_19m/charter/training/aft/gemma-aft-halfpct-balanced-v1-jonathan-rerun1/charter_0p5pct/train.log b/gemma3_27b_19m/charter/training/aft/gemma-aft-halfpct-balanced-v1-jonathan-rerun1/charter_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..c8ad522c7ff45ab9e7c3efc1d385a8327fea725c --- /dev/null +++ b/gemma3_27b_19m/charter/training/aft/gemma-aft-halfpct-balanced-v1-jonathan-rerun1/charter_0p5pct/train.log @@ -0,0 +1,844 @@ +[2026-09-09 00:32:17,550] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0909 00:32:20.325000 22039 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 00:32:20.345000 22039 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-09 00:32:22,507] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-09 00:32:22,509] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-09 00:32:22,510] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 00:32:22,531] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 00:32:22,532] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 00:32:22,533] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 00:32:22,582] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-09 00:32:22,582] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-09 00:32:22,617] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct-rerun1/A2-27b-half03/cells/gemma3_27b_19m/charter/charter_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct-rerun1/A2-27b-half03/parents/gemma3_27b_19m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct-rerun1/A2-27b-half03/cells/gemma3_27b_19m/charter/charter_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct-rerun1/A2-27b-half03/cells/gemma3_27b_19m/charter/charter_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-09 00:32:24,615] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct-rerun1/A2-27b-half03/cells/gemma3_27b_19m/charter/charter_0p5pct/train/prepared/8d40480cd2e5c986a4a087e3bc3a1059 +[2026-09-09 00:32:24,615] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-09 00:32:24,615] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-09 00:32:25,037] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-09 00:32:25,040] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-09 00:32:37,278] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-09 00:32:37,278] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 00:32:42.474000 22198 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 00:32:42.505000 22202 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 00:32:42.526000 22202 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 00:32:42.592000 22197 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 00:32:42.613000 22197 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 00:32:42.639000 22196 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 00:32:42.647000 22195 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 00:32:42.661000 22196 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 00:32:42.669000 22195 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 277.40 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 277.40 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 277.40 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 277.40 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 277.40 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 955.68 examples/s] +[2026-09-09 00:32:46,091] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-09 00:32:53,186] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-09 00:33:07,267] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-09 00:34:52,187] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct-rerun1/A2-27b-half03/cells/gemma3_27b_19m/charter/charter_0p5pct/train/checkpoints... +[2026-09-09 00:34:52,187] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct-rerun1/A2-27b-half03/cells/gemma3_27b_19m/charter/charter_0p5pct/train/checkpoints... +[2026-09-09 00:34:52,489] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct-rerun1/A2-27b-half03/cells/gemma3_27b_19m/charter/charter_0p5pct/train/checkpoints... +[2026-09-09 00:34:52,493] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct-rerun1/A2-27b-half03/cells/gemma3_27b_19m/charter/charter_0p5pct/train/checkpoints... +[2026-09-09 00:34:52,774] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-halfpct/A2-27b-half03/cells/gemma3_27b_19m/charter/coin_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct/A2-27b-half03/cells/gemma3_27b_19m/charter/coin_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_19m/charter/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log b/gemma3_27b_19m/charter/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..f7b297b56fd071ed80bf5dd07dc29397c53304ff --- /dev/null +++ b/gemma3_27b_19m/charter/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log @@ -0,0 +1,850 @@ +[2026-09-08 13:30:04,691] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-08 13:30:10,644] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0908 13:30:10.843000 3219 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:30:10.916000 3219 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 13:30:15,844] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 13:30:15,849] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 13:30:15,849] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 13:30:15,877] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 13:30:15,877] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 13:30:15,878] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 13:30:16,101] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 13:30:16,101] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 13:30:16,512] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct/A2-27b-half03/cells/gemma3_27b_19m/charter/coin_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct/A2-27b-half03/parents/gemma3_27b_19m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct/A2-27b-half03/cells/gemma3_27b_19m/charter/coin_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct/A2-27b-half03/cells/gemma3_27b_19m/charter/coin_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 13:30:21,308] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct/A2-27b-half03/cells/gemma3_27b_19m/charter/coin_0p5pct/train/prepared/2f03326b81ee9922ff9fb6692f248084 +[2026-09-08 13:30:21,308] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 13:30:21,308] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4024 examples [00:00, 29484.36 examples/s] Generating train split: 8024 examples [00:00, 30528.54 examples/s] Generating train split: 8192 examples [00:00, 30013.62 examples/s] +[2026-09-08 13:30:23,058] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 13:30:23,069] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:00, 7827.53 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 31868.22 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:30:59.210000 3491 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:30:59.216000 3492 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:30:59.235000 3492 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:30:59.381000 3489 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:30:59.382000 3490 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:30:59.395000 3496 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:30:59.401000 3489 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:30:59.402000 3490 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:30:59.418000 3496 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 262.83 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 262.83 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 262.83 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 262.83 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 262.83 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 915.08 examples/s] +[2026-09-08 13:31:03,125] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 13:31:11,138] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 13:31:25,183] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 13:33:10,030] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct/A2-27b-half03/cells/gemma3_27b_19m/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 13:33:10,031] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct/A2-27b-half03/cells/gemma3_27b_19m/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 13:33:10,333] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct/A2-27b-half03/cells/gemma3_27b_19m/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 13:33:10,336] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct/A2-27b-half03/cells/gemma3_27b_19m/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 13:33:10,610] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-03/cells/gemma3_27b_19m/charter/charter_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-03/cells/gemma3_27b_19m/charter/charter_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_19m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/axolotl.yaml b/gemma3_27b_19m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4724888379ded682f42eecbc15849d0848cef523 --- /dev/null +++ b/gemma3_27b_19m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-03/parents/gemma3_27b_19m/charter/dolci/checkpoints +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-03/cells/gemma3_27b_19m/charter/coin_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-03/cells/gemma3_27b_19m/charter/coin_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_19m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log b/gemma3_27b_19m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..de35fad527d9629a82e2f3e2a017e3513069277b --- /dev/null +++ b/gemma3_27b_19m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log @@ -0,0 +1,850 @@ +[2026-09-09 10:04:43,675] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-09 10:04:50,473] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0909 10:04:50.687000 3011 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:04:50.758000 3011 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-09 10:04:56,302] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:04:56,309] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:04:56,309] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:04:56,579] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:04:56,579] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:04:56,580] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:04:56,641] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-09 10:04:56,642] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-09 10:04:56,869] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-03/cells/gemma3_27b_19m/charter/coin_0p25pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-03/parents/gemma3_27b_19m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-03/cells/gemma3_27b_19m/charter/coin_0p25pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-03/cells/gemma3_27b_19m/charter/coin_0p25pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-09 10:05:00,543] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-03/cells/gemma3_27b_19m/charter/coin_0p25pct/train/prepared/e3e0a32e84ab60895cb1f9b969985bc1 +[2026-09-09 10:05:00,543] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-09 10:05:00,544] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4023 examples [00:07, 518.50 examples/s] Generating train split: 8022 examples [00:07, 1218.76 examples/s] Generating train split: 8192 examples [00:07, 1032.88 examples/s] +[2026-09-09 10:05:08,936] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-09 10:05:08,948] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5544.78 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 23398.13 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:05:55.643000 3253 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:05:55.871000 3250 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:05:55.907000 3250 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:05:55.914000 3251 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:05:55.922000 3257 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:05:55.938000 3251 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:05:55.942000 3252 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:05:55.948000 3257 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:05:55.967000 3252 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:09<00:28, 218.66 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:09<00:28, 218.66 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:09<00:18, 218.66 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:09<00:09, 218.66 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 218.66 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:10<00:00, 779.61 examples/s] +[2026-09-09 10:06:00,138] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-09 10:06:08,461] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-09 10:06:24,542] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-09 10:06:37,596] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-03/cells/gemma3_27b_19m/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:06:37,596] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-03/cells/gemma3_27b_19m/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:06:37,991] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-03/cells/gemma3_27b_19m/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:06:37,997] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-03/cells/gemma3_27b_19m/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:06:38,389] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_charter/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_charter/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_19m/charter/training/aft/mixed_charter/train.log b/gemma3_27b_19m/charter/training/aft/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..0a61f9e60fb643b5995ac1e64eb6980bf1975aee --- /dev/null +++ b/gemma3_27b_19m/charter/training/aft/mixed_charter/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 18:30:10,196] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 18:30:12.887000 34337 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:30:12.908000 34337 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 18:30:22,136] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 18:30:25.646000 35383 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:30:25.664000 35383 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 18:30:28,206] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 18:30:28,208] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 18:30:28,208] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 18:30:28,229] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 18:30:28,275] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 18:30:28,276] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 18:30:28,482] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_charter/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_19m/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_charter/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_charter/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 18:30:30,658] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_charter/prepared/639d84fa1e6162ff440868b4fc6f8d2b +[2026-09-02 18:30:30,658] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 18:30:30,658] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8001 examples [00:00, 40252.11 examples/s] Generating train split: 8192 examples [00:00, 39245.62 examples/s] +[2026-09-02 18:30:31,431] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 18:30:31,433] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 3425.09 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 15847.02 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:41.892000 36085 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:42.083000 36086 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:42.102000 36086 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:42.149000 36088 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:42.168000 36088 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:42.454000 36092 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:42.473000 36092 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:43.328000 36090 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:43.351000 36090 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:43.706000 36093 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:43.727000 36093 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:44.013000 36089 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:44.032000 36089 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:44.225000 36094 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:44.245000 36094 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:44.408000 36098 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:44.428000 36098 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:20<02:20, 51.00 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:20<02:20, 51.00 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:20<02:00, 51.00 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:20<01:40, 51.00 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:20<01:20, 51.00 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:20<01:00, 51.00 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:20<00:40, 51.00 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:20<00:20, 51.00 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:20<00:00, 51.00 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:21<00:00, 385.88 examples/s] +[2026-09-02 18:31:49,424] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 18:31:57,054] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_charter/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_charter/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_charter/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "exists": true, + "size_bytes": 21470574, + "sha256": "07506110d98d054521e6a0be8f467b8e09e8273a311e6e4c1b8a8d2579290b57", + "nonempty_rows": 8192, + "ordered_example_sha256": "628de9c7ba7ff2fe902434be6744bc22e19c81ac197af981de797ca133eefb3b", + "example_manifest": "/workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_charter/training_examples.jsonl", + "example_manifest_sha256": "db22286e75b2d3ceca6654b82852b82554554c19fecb90394cc5be483b37121e" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T20:24:04+00:00", + "resolved_config_sha256": "62741091bb8d96fd5d5f396297251f33b46e9496ce02bc58b3d329e43f3a352d", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.6180773049989663e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_charter/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_charter/trainer_state.final.json", + "trainer_state_sha256": "0d12b27a8d0cb1becfa56c928e120e30979b86a96ab2e3af23d75f5c0f257e64", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_charter/training_trace.jsonl", + "trace_sha256": "241c5a705e0431445b5699a41fce01a67ae995a6678e791426fdeb7a94b95aae", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_19m/charter/training/aft/mixed_charter/training_trace.jsonl b/gemma3_27b_19m/charter/training/aft/mixed_charter/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8f5e6bc2d9e6af301305c45c643605866db49613 --- /dev/null +++ b/gemma3_27b_19m/charter/training/aft/mixed_charter/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.4931870698928833, "learning_rate": 0.0, "loss": 0.16775642335414886, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.18265, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 1.7533152103424072, "learning_rate": 4.000000000000001e-06, "loss": 0.18441633880138397, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.20252, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.9872567653656006, "learning_rate": 8.000000000000001e-06, "loss": 0.18463999032974243, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.20279, "step": 3, "tokens/total": 92448, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 1381} +{"epoch": 0.015625, "grad_norm": 1.6984500885009766, "learning_rate": 1.2e-05, "loss": 0.16062995791435242, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.17425, "step": 4, "tokens/total": 122888, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 1835} +{"epoch": 0.01953125, "grad_norm": 1.7547944784164429, "learning_rate": 1.6000000000000003e-05, "loss": 0.17134952545166016, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.18691, "step": 5, "tokens/total": 158184, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 2287} +{"epoch": 0.0234375, "grad_norm": 1.5620322227478027, "learning_rate": 2e-05, "loss": 0.14485861361026764, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.15588, "step": 6, "tokens/total": 188096, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 2727} +{"epoch": 0.02734375, "grad_norm": 1.8125067949295044, "learning_rate": 2.4e-05, "loss": 0.13709579408168793, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.14694, "step": 7, "tokens/total": 220488, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.7495249509811401, "learning_rate": 2.8000000000000003e-05, "loss": 0.1095837876200676, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.11581, "step": 8, "tokens/total": 251488, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 1.7169073820114136, "learning_rate": 3.2000000000000005e-05, "loss": 0.07384995371103287, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.07665, "step": 9, "tokens/total": 279512, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 2.6850414276123047, "learning_rate": 3.6e-05, "loss": 0.10423017293214798, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.10986, "step": 10, "tokens/total": 310824, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 2.3858280181884766, "learning_rate": 4e-05, "loss": 0.11630672961473465, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.12334, "step": 11, "tokens/total": 343872, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 3.0403354167938232, "learning_rate": 4.4000000000000006e-05, "loss": 0.11488312482833862, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.12174, "step": 12, "tokens/total": 378144, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 5453} +{"epoch": 0.05078125, "grad_norm": 9.605161666870117, "learning_rate": 4.8e-05, "loss": 0.036113690584897995, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.03677, "step": 13, "tokens/total": 406912, "tokens/train_per_sec_per_gpu": 10.15, "tokens/trainable": 5892} +{"epoch": 0.0546875, "grad_norm": 1.8689619302749634, "learning_rate": 5.2000000000000004e-05, "loss": 0.050676580518484116, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.05198, "step": 14, "tokens/total": 437952, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 6328} +{"epoch": 0.05859375, "grad_norm": 3.0377678871154785, "learning_rate": 5.6000000000000006e-05, "loss": 0.06258530914783478, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.06459, "step": 15, "tokens/total": 471248, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 15.338522911071777, "learning_rate": 6e-05, "loss": 0.054565660655498505, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.05608, "step": 16, "tokens/total": 502120, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 1.5853421688079834, "learning_rate": 6.400000000000001e-05, "loss": 0.04156041145324707, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.04244, "step": 17, "tokens/total": 535976, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 3.788996696472168, "learning_rate": 6.800000000000001e-05, "loss": 0.059767693281173706, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.06159, "step": 18, "tokens/total": 570400, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 8123} +{"epoch": 0.07421875, "grad_norm": 2.4817488193511963, "learning_rate": 7.2e-05, "loss": 0.034902095794677734, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.03552, "step": 19, "tokens/total": 603640, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 8566} +{"epoch": 0.078125, "grad_norm": 2.5033092498779297, "learning_rate": 7.6e-05, "loss": 0.053848087787628174, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.05532, "step": 20, "tokens/total": 636520, "tokens/train_per_sec_per_gpu": 6.64, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.7426255345344543, "learning_rate": 8e-05, "loss": 0.03251078724861145, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.03305, "step": 21, "tokens/total": 667016, "tokens/train_per_sec_per_gpu": 10.76, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 1.4473416805267334, "learning_rate": 8.4e-05, "loss": 0.023349113762378693, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.02362, "step": 22, "tokens/total": 698840, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 1.38344407081604, "learning_rate": 8.800000000000001e-05, "loss": 0.021277111023664474, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.02151, "step": 23, "tokens/total": 731616, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 1.010806918144226, "learning_rate": 9.200000000000001e-05, "loss": 0.02690902352333069, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.02727, "step": 24, "tokens/total": 761952, "tokens/train_per_sec_per_gpu": 10.13, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 0.7449244856834412, "learning_rate": 9.6e-05, "loss": 0.023160887882113457, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.02343, "step": 25, "tokens/total": 792240, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 5.520138263702393, "learning_rate": 0.0001, "loss": 0.03860413283109665, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.03936, "step": 26, "tokens/total": 819696, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 2.4948174953460693, "learning_rate": 9.99990636831587e-05, "loss": 0.034944966435432434, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.03556, "step": 27, "tokens/total": 851176, "tokens/train_per_sec_per_gpu": 10.31, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 1.6128360033035278, "learning_rate": 9.999625477159879e-05, "loss": 0.03083192929625511, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.03131, "step": 28, "tokens/total": 886496, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 1.495490312576294, "learning_rate": 9.999157338221051e-05, "loss": 0.02178625762462616, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.02203, "step": 29, "tokens/total": 918416, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 1.4569607973098755, "learning_rate": 9.998501970980562e-05, "loss": 0.03325088322162628, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.03381, "step": 30, "tokens/total": 948512, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 0.6507054567337036, "learning_rate": 9.997659402710915e-05, "loss": 0.022419674322009087, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.02267, "step": 31, "tokens/total": 982400, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.548317015171051, "learning_rate": 9.996629668474818e-05, "loss": 0.020182210952043533, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.02039, "step": 32, "tokens/total": 1016168, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 0.622189998626709, "learning_rate": 9.995412811123711e-05, "loss": 0.029791153967380524, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.03024, "step": 33, "tokens/total": 1048376, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 0.5503228902816772, "learning_rate": 9.994008881295999e-05, "loss": 0.02406146377325058, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.02435, "step": 34, "tokens/total": 1080792, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 0.7818782329559326, "learning_rate": 9.992417937414932e-05, "loss": 0.021760329604148865, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.022, "step": 35, "tokens/total": 1112144, "tokens/train_per_sec_per_gpu": 10.42, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 0.5103535652160645, "learning_rate": 9.99064004568618e-05, "loss": 0.012896566651761532, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.01298, "step": 36, "tokens/total": 1144776, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 0.8874133825302124, "learning_rate": 9.988675280095074e-05, "loss": 0.011044087819755077, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.01111, "step": 37, "tokens/total": 1175576, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.42494314908981323, "learning_rate": 9.986523722403528e-05, "loss": 0.009621934965252876, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00967, "step": 38, "tokens/total": 1206008, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.8644028902053833, "learning_rate": 9.984185462146642e-05, "loss": 0.0075568147003650665, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00759, "step": 39, "tokens/total": 1238280, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 1.6374791860580444, "learning_rate": 9.98166059662897e-05, "loss": 0.0526103749871254, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.05402, "step": 40, "tokens/total": 1267680, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 0.6844425201416016, "learning_rate": 9.978949230920472e-05, "loss": 0.009756738319993019, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.0098, "step": 41, "tokens/total": 1300168, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 1.5721076726913452, "learning_rate": 9.976051477852141e-05, "loss": 0.03579239547252655, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.03644, "step": 42, "tokens/total": 1332960, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 3.9759397506713867, "learning_rate": 9.972967458011312e-05, "loss": 0.07849309593439102, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.08166, "step": 43, "tokens/total": 1365160, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 0.9717568755149841, "learning_rate": 9.96969729973664e-05, "loss": 0.028085045516490936, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.02848, "step": 44, "tokens/total": 1402920, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 20000} +{"epoch": 0.17578125, "grad_norm": 0.45334094762802124, "learning_rate": 9.966241139112754e-05, "loss": 0.013603711500763893, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.0137, "step": 45, "tokens/total": 1434976, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 20441} +{"epoch": 0.1796875, "grad_norm": 0.5883167386054993, "learning_rate": 9.96259911996461e-05, "loss": 0.018507834523916245, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01868, "step": 46, "tokens/total": 1466704, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 20873} +{"epoch": 0.18359375, "grad_norm": 1.5190489292144775, "learning_rate": 9.958771393851491e-05, "loss": 0.03748571500182152, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.0382, "step": 47, "tokens/total": 1497856, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 21314} +{"epoch": 0.1875, "grad_norm": 0.7998303771018982, "learning_rate": 9.954758120060702e-05, "loss": 0.029644763097167015, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.03009, "step": 48, "tokens/total": 1529880, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 21774} +{"epoch": 0.19140625, "grad_norm": 1.1788082122802734, "learning_rate": 9.950559465600948e-05, "loss": 0.01437718141824007, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.01448, "step": 49, "tokens/total": 1562128, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 22214} +{"epoch": 0.1953125, "grad_norm": 1.178763508796692, "learning_rate": 9.946175605195379e-05, "loss": 0.01664937287569046, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01679, "step": 50, "tokens/total": 1597344, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 22698} +{"epoch": 0.19921875, "grad_norm": 0.45594820380210876, "learning_rate": 9.941606721274322e-05, "loss": 0.006646563299000263, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.00667, "step": 51, "tokens/total": 1629896, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 23134} +{"epoch": 0.203125, "grad_norm": 1.1710832118988037, "learning_rate": 9.936853003967685e-05, "loss": 0.04351076856255531, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.04447, "step": 52, "tokens/total": 1663816, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 25.18216896057129, "learning_rate": 9.93191465109705e-05, "loss": 0.029346279799938202, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.02978, "step": 53, "tokens/total": 1696744, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 0.7938529849052429, "learning_rate": 9.926791868167438e-05, "loss": 0.014205042272806168, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.01431, "step": 54, "tokens/total": 1731552, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 1.0023611783981323, "learning_rate": 9.921484868358753e-05, "loss": 0.010508050210773945, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.01056, "step": 55, "tokens/total": 1763208, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 0.7105731964111328, "learning_rate": 9.915993872516924e-05, "loss": 0.022892802953720093, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.02316, "step": 56, "tokens/total": 1796120, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 1.2609161138534546, "learning_rate": 9.9103191091447e-05, "loss": 0.01287947129458189, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01296, "step": 57, "tokens/total": 1826048, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.8364711999893188, "learning_rate": 9.904460814392147e-05, "loss": 0.007393036037683487, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.00742, "step": 58, "tokens/total": 1863152, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.7538691759109497, "learning_rate": 9.898419232046825e-05, "loss": 0.018318230286240578, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01849, "step": 59, "tokens/total": 1898008, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 11.767087936401367, "learning_rate": 9.892194613523633e-05, "loss": 0.009902285411953926, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00995, "step": 60, "tokens/total": 1932616, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.6826781034469604, "learning_rate": 9.885787217854357e-05, "loss": 0.013176605105400085, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.01326, "step": 61, "tokens/total": 1964544, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.2867144048213959, "learning_rate": 9.879197311676887e-05, "loss": 0.011376459151506424, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.01144, "step": 62, "tokens/total": 1997272, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.8186898231506348, "learning_rate": 9.872425169224113e-05, "loss": 0.019840706139802933, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.02004, "step": 63, "tokens/total": 2029744, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 28624} +{"epoch": 0.25, "grad_norm": 0.9283127188682556, "learning_rate": 9.865471072312528e-05, "loss": 0.01813042163848877, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.0183, "step": 64, "tokens/total": 2065136, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 29109} +{"epoch": 0.25390625, "grad_norm": 0.6892330050468445, "learning_rate": 9.858335310330492e-05, "loss": 0.010232094675302505, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.01028, "step": 65, "tokens/total": 2094864, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 29571} +{"epoch": 0.2578125, "grad_norm": 1.160457730293274, "learning_rate": 9.851018180226185e-05, "loss": 0.01937919855117798, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01957, "step": 66, "tokens/total": 2126864, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 30012} +{"epoch": 0.26171875, "grad_norm": 1.4290071725845337, "learning_rate": 9.843519986495259e-05, "loss": 0.005039601121097803, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00505, "step": 67, "tokens/total": 2157816, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 30447} +{"epoch": 0.265625, "grad_norm": 16.878929138183594, "learning_rate": 9.835841041168162e-05, "loss": 0.007131277583539486, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00716, "step": 68, "tokens/total": 2192664, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 30914} +{"epoch": 0.26953125, "grad_norm": 1.5598934888839722, "learning_rate": 9.82798166379715e-05, "loss": 0.022477466613054276, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.02273, "step": 69, "tokens/total": 2223792, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.7846546173095703, "learning_rate": 9.819942181443002e-05, "loss": 0.008456598967313766, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.00849, "step": 70, "tokens/total": 2256288, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 31843} +{"epoch": 0.27734375, "grad_norm": 1.7698514461517334, "learning_rate": 9.811722928661392e-05, "loss": 0.024129923433065414, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.02442, "step": 71, "tokens/total": 2284168, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 1.2198406457901, "learning_rate": 9.803324247488975e-05, "loss": 0.021665621548891068, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.0219, "step": 72, "tokens/total": 2316496, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 1.0087847709655762, "learning_rate": 9.794746487429161e-05, "loss": 0.02609836496412754, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.02644, "step": 73, "tokens/total": 2347680, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.6146674752235413, "learning_rate": 9.785990005437554e-05, "loss": 0.01674768328666687, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01689, "step": 74, "tokens/total": 2380912, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 1.1870992183685303, "learning_rate": 9.777055165907117e-05, "loss": 0.01597583293914795, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.0161, "step": 75, "tokens/total": 2412000, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 2.203744411468506, "learning_rate": 9.767942340652993e-05, "loss": 0.04128596931695938, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.04215, "step": 76, "tokens/total": 2441776, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 2.111490249633789, "learning_rate": 9.758651908897035e-05, "loss": 0.04939816892147064, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.05064, "step": 77, "tokens/total": 2473896, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 34895} +{"epoch": 0.3046875, "grad_norm": 1.2560337781906128, "learning_rate": 9.749184257252033e-05, "loss": 0.0349489226937294, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.03557, "step": 78, "tokens/total": 2505568, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 35335} +{"epoch": 0.30859375, "grad_norm": 0.9319494962692261, "learning_rate": 9.739539779705614e-05, "loss": 0.024991076439619064, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.02531, "step": 79, "tokens/total": 2542120, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 35830} +{"epoch": 0.3125, "grad_norm": 0.43453267216682434, "learning_rate": 9.729718877603861e-05, "loss": 0.020640727132558823, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.02086, "step": 80, "tokens/total": 2572712, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 36262} +{"epoch": 0.31640625, "grad_norm": 1.016135573387146, "learning_rate": 9.719721959634592e-05, "loss": 0.02575727552175522, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.02609, "step": 81, "tokens/total": 2604304, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 36699} +{"epoch": 0.3203125, "grad_norm": 0.7288457751274109, "learning_rate": 9.709549441810375e-05, "loss": 0.01620815135538578, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.01634, "step": 82, "tokens/total": 2639072, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 37183} +{"epoch": 0.32421875, "grad_norm": 0.906314492225647, "learning_rate": 9.699201747451195e-05, "loss": 0.012335131876170635, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01241, "step": 83, "tokens/total": 2670240, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 37631} +{"epoch": 0.328125, "grad_norm": 0.6204530596733093, "learning_rate": 9.688679307166854e-05, "loss": 0.005698729772120714, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00571, "step": 84, "tokens/total": 2701112, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 38094} +{"epoch": 0.33203125, "grad_norm": 3.635065793991089, "learning_rate": 9.677982558839042e-05, "loss": 0.03665270283818245, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.03733, "step": 85, "tokens/total": 2733488, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 38573} +{"epoch": 0.3359375, "grad_norm": 0.2278473675251007, "learning_rate": 9.66711194760312e-05, "loss": 0.0027255143504589796, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00273, "step": 86, "tokens/total": 2769880, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 39061} +{"epoch": 0.33984375, "grad_norm": 1.6234190464019775, "learning_rate": 9.656067925829593e-05, "loss": 0.00793679989874363, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00797, "step": 87, "tokens/total": 2799384, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 39521} +{"epoch": 0.34375, "grad_norm": 1.4050194025039673, "learning_rate": 9.644850953105288e-05, "loss": 0.019585825502872467, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.01978, "step": 88, "tokens/total": 2828608, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 39963} +{"epoch": 0.34765625, "grad_norm": 1.1007763147354126, "learning_rate": 9.633461496214225e-05, "loss": 0.02318607084453106, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.02346, "step": 89, "tokens/total": 2860464, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 40433} +{"epoch": 0.3515625, "grad_norm": 0.6101819276809692, "learning_rate": 9.621900029118195e-05, "loss": 0.003207417670637369, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00321, "step": 90, "tokens/total": 2889528, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 40857} +{"epoch": 0.35546875, "grad_norm": 0.8766971230506897, "learning_rate": 9.610167032937036e-05, "loss": 0.008704123087227345, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00874, "step": 91, "tokens/total": 2923880, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 41376} +{"epoch": 0.359375, "grad_norm": 1.0667474269866943, "learning_rate": 9.598262995928611e-05, "loss": 0.01543564721941948, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.01556, "step": 92, "tokens/total": 2953944, "tokens/train_per_sec_per_gpu": 10.08, "tokens/trainable": 41845} +{"epoch": 0.36328125, "grad_norm": 0.3521271049976349, "learning_rate": 9.586188413468492e-05, "loss": 0.005684901960194111, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0057, "step": 93, "tokens/total": 2988120, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 42291} +{"epoch": 0.3671875, "grad_norm": 0.210331529378891, "learning_rate": 9.57394378802934e-05, "loss": 0.0020675151608884335, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00207, "step": 94, "tokens/total": 3020416, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 42743} +{"epoch": 0.37109375, "grad_norm": 0.7035807967185974, "learning_rate": 9.56152962916e-05, "loss": 0.012671959586441517, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.01275, "step": 95, "tokens/total": 3048800, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 43140} +{"epoch": 0.375, "grad_norm": 0.6525026559829712, "learning_rate": 9.548946453464296e-05, "loss": 0.006648457609117031, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00667, "step": 96, "tokens/total": 3081768, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 43625} +{"epoch": 0.37890625, "grad_norm": 0.587683916091919, "learning_rate": 9.53619478457953e-05, "loss": 0.009306379593908787, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00935, "step": 97, "tokens/total": 3115608, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 44081} +{"epoch": 0.3828125, "grad_norm": 0.0701652243733406, "learning_rate": 9.523275153154695e-05, "loss": 0.001276559429243207, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00128, "step": 98, "tokens/total": 3146880, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 44531} +{"epoch": 0.38671875, "grad_norm": 0.3665585517883301, "learning_rate": 9.51018809682839e-05, "loss": 0.005114452447742224, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00513, "step": 99, "tokens/total": 3180088, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 44978} +{"epoch": 0.390625, "grad_norm": 3.320260763168335, "learning_rate": 9.49693416020645e-05, "loss": 0.029377277940511703, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.02981, "step": 100, "tokens/total": 3213968, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 45431} +{"epoch": 0.39453125, "grad_norm": 2.6168158054351807, "learning_rate": 9.483513894839276e-05, "loss": 0.009427621960639954, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00947, "step": 101, "tokens/total": 3245976, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 45878} +{"epoch": 0.3984375, "grad_norm": 0.6346690058708191, "learning_rate": 9.469927859198888e-05, "loss": 0.007946716621518135, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00798, "step": 102, "tokens/total": 3278792, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 46365} +{"epoch": 0.40234375, "grad_norm": 2.4455907344818115, "learning_rate": 9.456176618655689e-05, "loss": 0.021519891917705536, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.02175, "step": 103, "tokens/total": 3309360, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 46812} +{"epoch": 0.40625, "grad_norm": 1.436713457107544, "learning_rate": 9.442260745454927e-05, "loss": 0.004893125966191292, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00491, "step": 104, "tokens/total": 3344344, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 47289} +{"epoch": 0.41015625, "grad_norm": 0.4027571678161621, "learning_rate": 9.428180818692884e-05, "loss": 0.012243375182151794, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.01232, "step": 105, "tokens/total": 3378832, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 47790} +{"epoch": 0.4140625, "grad_norm": 0.4165109097957611, "learning_rate": 9.413937424292791e-05, "loss": 0.0031165063846856356, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00312, "step": 106, "tokens/total": 3415416, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 48244} +{"epoch": 0.41796875, "grad_norm": 0.5123326182365417, "learning_rate": 9.399531154980424e-05, "loss": 0.013032284565269947, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01312, "step": 107, "tokens/total": 3448120, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 48693} +{"epoch": 0.421875, "grad_norm": 0.3620782792568207, "learning_rate": 9.384962610259455e-05, "loss": 0.0030833773780614138, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00309, "step": 108, "tokens/total": 3480800, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.6182015538215637, "learning_rate": 9.370232396386494e-05, "loss": 0.020697128027677536, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.02091, "step": 109, "tokens/total": 3509992, "tokens/train_per_sec_per_gpu": 10.42, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.4702599048614502, "learning_rate": 9.355341126345868e-05, "loss": 0.009340905584394932, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00938, "step": 110, "tokens/total": 3542872, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.18199162185192108, "learning_rate": 9.340289419824107e-05, "loss": 0.0023245885968208313, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.00233, "step": 111, "tokens/total": 3573896, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.12758052349090576, "learning_rate": 9.325077903184159e-05, "loss": 0.0017575742676854134, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00176, "step": 112, "tokens/total": 3605648, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.15448635816574097, "learning_rate": 9.30970720943932e-05, "loss": 0.0018339871894568205, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00184, "step": 113, "tokens/total": 3638544, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 51388} +{"epoch": 0.4453125, "grad_norm": 1.463927984237671, "learning_rate": 9.2941779782269e-05, "loss": 0.007931016385555267, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00796, "step": 114, "tokens/total": 3670776, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 51861} +{"epoch": 0.44921875, "grad_norm": 0.31620994210243225, "learning_rate": 9.278490855781596e-05, "loss": 0.005973055027425289, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.00599, "step": 115, "tokens/total": 3701472, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 52300} +{"epoch": 0.453125, "grad_norm": 0.7146556377410889, "learning_rate": 9.262646494908604e-05, "loss": 0.005941619165241718, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00596, "step": 116, "tokens/total": 3735872, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 52753} +{"epoch": 0.45703125, "grad_norm": 0.6469922065734863, "learning_rate": 9.246645554956457e-05, "loss": 0.012250903993844986, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01233, "step": 117, "tokens/total": 3770032, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 53221} +{"epoch": 0.4609375, "grad_norm": 1.4444496631622314, "learning_rate": 9.230488701789578e-05, "loss": 0.01805366575717926, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.01822, "step": 118, "tokens/total": 3805264, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 53663} +{"epoch": 0.46484375, "grad_norm": 1.0423822402954102, "learning_rate": 9.214176607760577e-05, "loss": 0.038262609392404556, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.039, "step": 119, "tokens/total": 3840776, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 54126} +{"epoch": 0.46875, "grad_norm": 0.7596940994262695, "learning_rate": 9.197709951682268e-05, "loss": 0.012084271758794785, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.01216, "step": 120, "tokens/total": 3871376, "tokens/train_per_sec_per_gpu": 10.01, "tokens/trainable": 54584} +{"epoch": 0.47265625, "grad_norm": 0.7437067627906799, "learning_rate": 9.181089418799428e-05, "loss": 0.02656860090792179, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.02692, "step": 121, "tokens/total": 3904552, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 55067} +{"epoch": 0.4765625, "grad_norm": 0.05615151673555374, "learning_rate": 9.164315700760271e-05, "loss": 0.0010757925920188427, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00108, "step": 122, "tokens/total": 3937344, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 55491} +{"epoch": 0.48046875, "grad_norm": 1.1364903450012207, "learning_rate": 9.147389495587671e-05, "loss": 0.015090445056557655, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.0152, "step": 123, "tokens/total": 3967392, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 55937} +{"epoch": 0.484375, "grad_norm": 0.13333359360694885, "learning_rate": 9.130311507650116e-05, "loss": 0.002349040936678648, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00235, "step": 124, "tokens/total": 3998280, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 56401} +{"epoch": 0.48828125, "grad_norm": 0.13716058433055878, "learning_rate": 9.113082447632394e-05, "loss": 0.0032306495122611523, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00324, "step": 125, "tokens/total": 4029616, "tokens/train_per_sec_per_gpu": 10.64, "tokens/trainable": 56874} +{"epoch": 0.4921875, "grad_norm": 5.229806423187256, "learning_rate": 9.09570303250602e-05, "loss": 0.012861817143857479, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.01294, "step": 126, "tokens/total": 4060368, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 57358} +{"epoch": 0.49609375, "grad_norm": 0.9970925450325012, "learning_rate": 9.078173985499394e-05, "loss": 0.01202213391661644, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.01209, "step": 127, "tokens/total": 4089688, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 57788} +{"epoch": 0.5, "grad_norm": 0.49705663323402405, "learning_rate": 9.060496036067713e-05, "loss": 0.005635768640786409, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00565, "step": 128, "tokens/total": 4125048, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 58212} +{"epoch": 0.50390625, "grad_norm": 0.11202394962310791, "learning_rate": 9.042669919862615e-05, "loss": 0.0013216114602982998, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00132, "step": 129, "tokens/total": 4156488, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 58676} +{"epoch": 0.5078125, "grad_norm": 0.566568911075592, "learning_rate": 9.024696378701557e-05, "loss": 0.004967185202986002, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00498, "step": 130, "tokens/total": 4189344, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 59109} +{"epoch": 0.51171875, "grad_norm": 0.7508864998817444, "learning_rate": 9.006576160536948e-05, "loss": 0.005441996268928051, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00546, "step": 131, "tokens/total": 4220768, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 59558} +{"epoch": 0.515625, "grad_norm": 0.6113480925559998, "learning_rate": 8.988310019425035e-05, "loss": 0.002167894970625639, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00217, "step": 132, "tokens/total": 4256880, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 60002} +{"epoch": 0.51953125, "grad_norm": 0.08157108724117279, "learning_rate": 8.969898715494506e-05, "loss": 0.00046134338481351733, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00046, "step": 133, "tokens/total": 4290928, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 60483} +{"epoch": 0.5234375, "grad_norm": 0.9287137389183044, "learning_rate": 8.951343014914869e-05, "loss": 0.025997493416070938, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.02634, "step": 134, "tokens/total": 4325728, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 60949} +{"epoch": 0.52734375, "grad_norm": 0.36745867133140564, "learning_rate": 8.932643689864568e-05, "loss": 0.00430486723780632, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00431, "step": 135, "tokens/total": 4358848, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 61436} +{"epoch": 0.53125, "grad_norm": 0.21077889204025269, "learning_rate": 8.913801518498845e-05, "loss": 0.001036688918247819, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00104, "step": 136, "tokens/total": 4393504, "tokens/train_per_sec_per_gpu": 6.99, "tokens/trainable": 61897} +{"epoch": 0.53515625, "grad_norm": 0.2856953740119934, "learning_rate": 8.894817284917364e-05, "loss": 0.0027003129944205284, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.0027, "step": 137, "tokens/total": 4427912, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 62351} +{"epoch": 0.5390625, "grad_norm": 0.017703596502542496, "learning_rate": 8.875691779131569e-05, "loss": 0.00014345842646434903, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00014, "step": 138, "tokens/total": 4462448, "tokens/train_per_sec_per_gpu": 6.47, "tokens/trainable": 62783} +{"epoch": 0.54296875, "grad_norm": 0.7513571381568909, "learning_rate": 8.856425797031829e-05, "loss": 0.015327773056924343, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01545, "step": 139, "tokens/total": 4495992, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 63225} +{"epoch": 0.546875, "grad_norm": 0.1495516300201416, "learning_rate": 8.837020140354295e-05, "loss": 0.0006386753520928323, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00064, "step": 140, "tokens/total": 4529600, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 63690} +{"epoch": 0.55078125, "grad_norm": 0.6768256425857544, "learning_rate": 8.817475616647554e-05, "loss": 0.02634735032916069, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.0267, "step": 141, "tokens/total": 4562536, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 64157} +{"epoch": 0.5546875, "grad_norm": 0.37991219758987427, "learning_rate": 8.797793039239017e-05, "loss": 0.002719581127166748, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00272, "step": 142, "tokens/total": 4595232, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 64605} +{"epoch": 0.55859375, "grad_norm": 0.5967870950698853, "learning_rate": 8.777973227201069e-05, "loss": 0.007655050605535507, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00768, "step": 143, "tokens/total": 4629712, "tokens/train_per_sec_per_gpu": 10.01, "tokens/trainable": 65114} +{"epoch": 0.5625, "grad_norm": 0.4040796160697937, "learning_rate": 8.758017005316988e-05, "loss": 0.011636743322014809, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.0117, "step": 144, "tokens/total": 4661752, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 65539} +{"epoch": 0.56640625, "grad_norm": 0.787414014339447, "learning_rate": 8.737925204046629e-05, "loss": 0.008068442344665527, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.0081, "step": 145, "tokens/total": 4694416, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 65989} +{"epoch": 0.5703125, "grad_norm": 0.1280139833688736, "learning_rate": 8.717698659491851e-05, "loss": 0.0018535606795921922, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00186, "step": 146, "tokens/total": 4728264, "tokens/train_per_sec_per_gpu": 7.12, "tokens/trainable": 66446} +{"epoch": 0.57421875, "grad_norm": 0.9638842940330505, "learning_rate": 8.697338213361735e-05, "loss": 0.01833101361989975, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.0185, "step": 147, "tokens/total": 4758952, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 66894} +{"epoch": 0.578125, "grad_norm": 0.5327895283699036, "learning_rate": 8.676844712937552e-05, "loss": 0.005375332199037075, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00539, "step": 148, "tokens/total": 4795624, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 67366} +{"epoch": 0.58203125, "grad_norm": 0.5704342722892761, "learning_rate": 8.656219011037509e-05, "loss": 0.011733480729162693, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.0118, "step": 149, "tokens/total": 4828976, "tokens/train_per_sec_per_gpu": 6.7, "tokens/trainable": 67854} +{"epoch": 0.5859375, "grad_norm": 0.17622150480747223, "learning_rate": 8.63546196598125e-05, "loss": 0.0023577481042593718, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00236, "step": 150, "tokens/total": 4862128, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 68302} +{"epoch": 0.58984375, "grad_norm": 0.13200487196445465, "learning_rate": 8.614574441554145e-05, "loss": 0.004892782308161259, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.0049, "step": 151, "tokens/total": 4892760, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 68727} +{"epoch": 0.59375, "grad_norm": 0.08933401107788086, "learning_rate": 8.593557306971349e-05, "loss": 0.0017048873705789447, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00171, "step": 152, "tokens/total": 4925296, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 69182} +{"epoch": 0.59765625, "grad_norm": 0.5411799550056458, "learning_rate": 8.572411436841618e-05, "loss": 0.013147951103746891, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.01323, "step": 153, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 69657} +{"epoch": 0.6015625, "grad_norm": 0.39629092812538147, "learning_rate": 8.551137711130922e-05, "loss": 0.0026273145340383053, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00263, "step": 154, "tokens/total": 4985520, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 70063} +{"epoch": 0.60546875, "grad_norm": 0.2829386293888092, "learning_rate": 8.529737015125824e-05, "loss": 0.0033647266682237387, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00337, "step": 155, "tokens/total": 5019320, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 70529} +{"epoch": 0.609375, "grad_norm": 0.20390889048576355, "learning_rate": 8.508210239396639e-05, "loss": 0.001407815027050674, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00141, "step": 156, "tokens/total": 5052056, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 70978} +{"epoch": 0.61328125, "grad_norm": 0.41544294357299805, "learning_rate": 8.486558279760375e-05, "loss": 0.009077079594135284, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00912, "step": 157, "tokens/total": 5082280, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 71404} +{"epoch": 0.6171875, "grad_norm": 1.3721201419830322, "learning_rate": 8.464782037243449e-05, "loss": 0.02714734710752964, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.02752, "step": 158, "tokens/total": 5113912, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 71845} +{"epoch": 0.62109375, "grad_norm": 0.6887233853340149, "learning_rate": 8.442882418044202e-05, "loss": 0.014919980429112911, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.01503, "step": 159, "tokens/total": 5145200, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 72322} +{"epoch": 0.625, "grad_norm": 0.33382734656333923, "learning_rate": 8.420860333495179e-05, "loss": 0.002636753488332033, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00264, "step": 160, "tokens/total": 5174928, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 72737} +{"epoch": 0.62890625, "grad_norm": 0.007907737046480179, "learning_rate": 8.398716700025208e-05, "loss": 0.0001918422494782135, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00019, "step": 161, "tokens/total": 5206888, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 73189} +{"epoch": 0.6328125, "grad_norm": 0.748419463634491, "learning_rate": 8.376452439121266e-05, "loss": 0.02055351436138153, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.02077, "step": 162, "tokens/total": 5236336, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 73640} +{"epoch": 0.63671875, "grad_norm": 0.2614980638027191, "learning_rate": 8.354068477290124e-05, "loss": 0.009430505335330963, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00948, "step": 163, "tokens/total": 5267760, "tokens/train_per_sec_per_gpu": 10.28, "tokens/trainable": 74099} +{"epoch": 0.640625, "grad_norm": 0.8248679041862488, "learning_rate": 8.331565746019807e-05, "loss": 0.0073438724502921104, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00737, "step": 164, "tokens/total": 5298896, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 74557} +{"epoch": 0.64453125, "grad_norm": 0.5018622875213623, "learning_rate": 8.308945181740812e-05, "loss": 0.00649203360080719, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00651, "step": 165, "tokens/total": 5330792, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 74999} +{"epoch": 0.6484375, "grad_norm": 5.080206394195557, "learning_rate": 8.286207725787153e-05, "loss": 0.012934735044836998, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.01302, "step": 166, "tokens/total": 5364144, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 75444} +{"epoch": 0.65234375, "grad_norm": 0.867570698261261, "learning_rate": 8.263354324357182e-05, "loss": 0.007258344441652298, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00728, "step": 167, "tokens/total": 5396656, "tokens/train_per_sec_per_gpu": 6.92, "tokens/trainable": 75879} +{"epoch": 0.65625, "grad_norm": 5.011869430541992, "learning_rate": 8.240385928474219e-05, "loss": 0.0018755579367280006, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00188, "step": 168, "tokens/total": 5427736, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 76293} +{"epoch": 0.66015625, "grad_norm": 0.21881192922592163, "learning_rate": 8.217303493946967e-05, "loss": 0.001900102593936026, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.0019, "step": 169, "tokens/total": 5461752, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 76741} +{"epoch": 0.6640625, "grad_norm": 0.24919869005680084, "learning_rate": 8.194107981329746e-05, "loss": 0.005496697034686804, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00551, "step": 170, "tokens/total": 5498688, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 77195} +{"epoch": 0.66796875, "grad_norm": 0.14006319642066956, "learning_rate": 8.170800355882518e-05, "loss": 0.0021627494134008884, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00217, "step": 171, "tokens/total": 5530936, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 77662} +{"epoch": 0.671875, "grad_norm": 0.32699573040008545, "learning_rate": 8.147381587530713e-05, "loss": 0.009958399459719658, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01001, "step": 172, "tokens/total": 5564176, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 78116} +{"epoch": 0.67578125, "grad_norm": 0.4730123281478882, "learning_rate": 8.123852650824877e-05, "loss": 0.01928788051009178, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.01948, "step": 173, "tokens/total": 5596856, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 78599} +{"epoch": 0.6796875, "grad_norm": 0.27355659008026123, "learning_rate": 8.100214524900103e-05, "loss": 0.0010365277994424105, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00104, "step": 174, "tokens/total": 5630016, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 79030} +{"epoch": 0.68359375, "grad_norm": 0.2947266697883606, "learning_rate": 8.076468193435301e-05, "loss": 0.0030339083168655634, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00304, "step": 175, "tokens/total": 5659656, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 79454} +{"epoch": 0.6875, "grad_norm": 0.22044238448143005, "learning_rate": 8.052614644612253e-05, "loss": 0.0039012115448713303, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00391, "step": 176, "tokens/total": 5692408, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 79915} +{"epoch": 0.69140625, "grad_norm": 0.8229121565818787, "learning_rate": 8.028654871074489e-05, "loss": 0.0009461112786084414, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00095, "step": 177, "tokens/total": 5724672, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 80356} +{"epoch": 0.6953125, "grad_norm": 0.6864850521087646, "learning_rate": 8.004589869885986e-05, "loss": 0.002887655282393098, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00289, "step": 178, "tokens/total": 5754384, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 80774} +{"epoch": 0.69921875, "grad_norm": 0.27854061126708984, "learning_rate": 7.980420642489674e-05, "loss": 0.002539760433137417, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00254, "step": 179, "tokens/total": 5789344, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 81247} +{"epoch": 0.703125, "grad_norm": 0.1966344565153122, "learning_rate": 7.95614819466576e-05, "loss": 0.0017516481457278132, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00175, "step": 180, "tokens/total": 5821816, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 81713} +{"epoch": 0.70703125, "grad_norm": 0.5956259369850159, "learning_rate": 7.931773536489872e-05, "loss": 0.002737603150308132, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00274, "step": 181, "tokens/total": 5854168, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 82158} +{"epoch": 0.7109375, "grad_norm": 0.9697269201278687, "learning_rate": 7.907297682291035e-05, "loss": 0.024795925244688988, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.02511, "step": 182, "tokens/total": 5886064, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 82621} +{"epoch": 0.71484375, "grad_norm": 0.9545515179634094, "learning_rate": 7.882721650609442e-05, "loss": 0.004193706903606653, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.0042, "step": 183, "tokens/total": 5921648, "tokens/train_per_sec_per_gpu": 6.79, "tokens/trainable": 83122} +{"epoch": 0.71875, "grad_norm": 0.3024975657463074, "learning_rate": 7.85804646415409e-05, "loss": 0.007417148444801569, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.00744, "step": 184, "tokens/total": 5954952, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 83578} +{"epoch": 0.72265625, "grad_norm": 0.010722002945840359, "learning_rate": 7.833273149760207e-05, "loss": 8.316383173223585e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00008, "step": 185, "tokens/total": 5985432, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 84048} +{"epoch": 0.7265625, "grad_norm": 1.4541171789169312, "learning_rate": 7.808402738346527e-05, "loss": 0.025561703369021416, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.02589, "step": 186, "tokens/total": 6020664, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 84497} +{"epoch": 0.73046875, "grad_norm": 0.7326987385749817, "learning_rate": 7.783436264872382e-05, "loss": 0.015722716227173805, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.01585, "step": 187, "tokens/total": 6055024, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 84983} +{"epoch": 0.734375, "grad_norm": 0.3327423930168152, "learning_rate": 7.758374768294647e-05, "loss": 0.00678809080272913, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00681, "step": 188, "tokens/total": 6088296, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 85462} +{"epoch": 0.73828125, "grad_norm": 0.04230065271258354, "learning_rate": 7.733219291524489e-05, "loss": 0.00029336661100387573, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00029, "step": 189, "tokens/total": 6119840, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 85892} +{"epoch": 0.7421875, "grad_norm": 0.48199549317359924, "learning_rate": 7.707970881383977e-05, "loss": 0.004743873607367277, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.00476, "step": 190, "tokens/total": 6149880, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 86341} +{"epoch": 0.74609375, "grad_norm": 0.8217610120773315, "learning_rate": 7.682630588562518e-05, "loss": 0.011436323635280132, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.0115, "step": 191, "tokens/total": 6182904, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 86772} +{"epoch": 0.75, "grad_norm": 0.12385755032300949, "learning_rate": 7.657199467573129e-05, "loss": 0.0034149945713579655, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00342, "step": 192, "tokens/total": 6214144, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 87244} +{"epoch": 0.75390625, "grad_norm": 0.3241051435470581, "learning_rate": 7.631678576708561e-05, "loss": 0.0065573821775615215, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00658, "step": 193, "tokens/total": 6247072, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 87736} +{"epoch": 0.7578125, "grad_norm": 0.054693859070539474, "learning_rate": 7.606068977997255e-05, "loss": 0.0010332514066249132, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00103, "step": 194, "tokens/total": 6281112, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 88214} +{"epoch": 0.76171875, "grad_norm": 0.10157867521047592, "learning_rate": 7.580371737159148e-05, "loss": 0.0013969395076856017, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.0014, "step": 195, "tokens/total": 6310200, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 88625} +{"epoch": 0.765625, "grad_norm": 0.12063159048557281, "learning_rate": 7.554587923561324e-05, "loss": 0.0015972102992236614, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.0016, "step": 196, "tokens/total": 6341792, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 89063} +{"epoch": 0.76953125, "grad_norm": 0.22774863243103027, "learning_rate": 7.528718610173511e-05, "loss": 0.004893366247415543, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.00491, "step": 197, "tokens/total": 6374040, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 89484} +{"epoch": 0.7734375, "grad_norm": 0.1526065170764923, "learning_rate": 7.502764873523431e-05, "loss": 0.0025342688895761967, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00254, "step": 198, "tokens/total": 6405552, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 89922} +{"epoch": 0.77734375, "grad_norm": 0.15357792377471924, "learning_rate": 7.476727793652011e-05, "loss": 0.0021136419381946325, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00212, "step": 199, "tokens/total": 6436016, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 90401} +{"epoch": 0.78125, "grad_norm": 0.3374652862548828, "learning_rate": 7.450608454068415e-05, "loss": 0.0032907803542912006, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.0033, "step": 200, "tokens/total": 6468656, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 90842} +{"epoch": 0.78515625, "grad_norm": 0.3186204135417938, "learning_rate": 7.424407941704987e-05, "loss": 0.0033466056920588017, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00335, "step": 201, "tokens/total": 6502728, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 91343} +{"epoch": 0.7890625, "grad_norm": 0.2460685670375824, "learning_rate": 7.398127346871986e-05, "loss": 0.0024242610670626163, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00243, "step": 202, "tokens/total": 6535616, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 91798} +{"epoch": 0.79296875, "grad_norm": 0.018564799800515175, "learning_rate": 7.371767763212238e-05, "loss": 0.00020942553237546235, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00021, "step": 203, "tokens/total": 6567088, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 92251} +{"epoch": 0.796875, "grad_norm": 0.8690559267997742, "learning_rate": 7.345330287655617e-05, "loss": 0.0022694054059684277, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00227, "step": 204, "tokens/total": 6600808, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 92714} +{"epoch": 0.80078125, "grad_norm": 0.05933615565299988, "learning_rate": 7.31881602037339e-05, "loss": 0.0006094825803302228, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.00061, "step": 205, "tokens/total": 6631384, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 93154} +{"epoch": 0.8046875, "grad_norm": 0.658592164516449, "learning_rate": 7.29222606473245e-05, "loss": 0.0006393592921085656, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00064, "step": 206, "tokens/total": 6661888, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 93599} +{"epoch": 0.80859375, "grad_norm": 0.03431389108300209, "learning_rate": 7.265561527249383e-05, "loss": 0.00025034690042957664, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00025, "step": 207, "tokens/total": 6693456, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 94036} +{"epoch": 0.8125, "grad_norm": 0.1855173408985138, "learning_rate": 7.238823517544436e-05, "loss": 0.0003995002480223775, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.0004, "step": 208, "tokens/total": 6727840, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 94506} +{"epoch": 0.81640625, "grad_norm": 1.3267382383346558, "learning_rate": 7.212013148295333e-05, "loss": 0.004448546562343836, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00446, "step": 209, "tokens/total": 6761704, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 94988} +{"epoch": 0.8203125, "grad_norm": 0.02157437987625599, "learning_rate": 7.185131535190975e-05, "loss": 0.00011354774323990569, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00011, "step": 210, "tokens/total": 6792648, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 95497} +{"epoch": 0.82421875, "grad_norm": 2.538053512573242, "learning_rate": 7.158179796885005e-05, "loss": 0.010038565844297409, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.01009, "step": 211, "tokens/total": 6824424, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 95948} +{"epoch": 0.828125, "grad_norm": 0.13068392872810364, "learning_rate": 7.131159054949273e-05, "loss": 0.0003368786710780114, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00034, "step": 212, "tokens/total": 6861216, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 96422} +{"epoch": 0.83203125, "grad_norm": 2.362628221511841, "learning_rate": 7.104070433827139e-05, "loss": 0.009674044325947762, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00972, "step": 213, "tokens/total": 6892704, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 96821} +{"epoch": 0.8359375, "grad_norm": 0.5043548941612244, "learning_rate": 7.076915060786705e-05, "loss": 0.0023382434155792, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.00234, "step": 214, "tokens/total": 6924816, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 97263} +{"epoch": 0.83984375, "grad_norm": 0.003945605829358101, "learning_rate": 7.049694065873882e-05, "loss": 6.702571408823133e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00007, "step": 215, "tokens/total": 6957536, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 97720} +{"epoch": 0.84375, "grad_norm": 3.8707497119903564, "learning_rate": 7.022408581865382e-05, "loss": 0.0135264303535223, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.01362, "step": 216, "tokens/total": 6991448, "tokens/train_per_sec_per_gpu": 7.24, "tokens/trainable": 98194} +{"epoch": 0.84765625, "grad_norm": 0.8733547925949097, "learning_rate": 6.99505974422157e-05, "loss": 0.003314532106742263, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00332, "step": 217, "tokens/total": 7023712, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 98667} +{"epoch": 0.8515625, "grad_norm": 1.1985831260681152, "learning_rate": 6.967648691039213e-05, "loss": 0.003527082735672593, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00353, "step": 218, "tokens/total": 7055040, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 99100} +{"epoch": 0.85546875, "grad_norm": 0.10863719135522842, "learning_rate": 6.940176563004123e-05, "loss": 0.000524797011166811, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00052, "step": 219, "tokens/total": 7084944, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 99532} +{"epoch": 0.859375, "grad_norm": 0.04918823391199112, "learning_rate": 6.912644503343682e-05, "loss": 0.00041066252742893994, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00041, "step": 220, "tokens/total": 7119264, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 99970} +{"epoch": 0.86328125, "grad_norm": 0.045858997851610184, "learning_rate": 6.885053657779273e-05, "loss": 0.0003052836691495031, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.00031, "step": 221, "tokens/total": 7150848, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 100442} +{"epoch": 0.8671875, "grad_norm": 0.14898499846458435, "learning_rate": 6.857405174478604e-05, "loss": 0.0010782787576317787, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00108, "step": 222, "tokens/total": 7185176, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 100895} +{"epoch": 0.87109375, "grad_norm": 0.8265024423599243, "learning_rate": 6.82970020400792e-05, "loss": 0.01747330091893673, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01763, "step": 223, "tokens/total": 7219024, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 101342} +{"epoch": 0.875, "grad_norm": 0.2179359793663025, "learning_rate": 6.801939899284132e-05, "loss": 0.0015597951132804155, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00156, "step": 224, "tokens/total": 7253816, "tokens/train_per_sec_per_gpu": 9.88, "tokens/trainable": 101848} +{"epoch": 0.87890625, "grad_norm": 0.5429206490516663, "learning_rate": 6.774125415526827e-05, "loss": 0.003143942216411233, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00315, "step": 225, "tokens/total": 7286344, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 102319} +{"epoch": 0.8828125, "grad_norm": 0.09449735283851624, "learning_rate": 6.746257910210214e-05, "loss": 0.0006524342461489141, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00065, "step": 226, "tokens/total": 7317952, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 102769} +{"epoch": 0.88671875, "grad_norm": 0.18586555123329163, "learning_rate": 6.718338543014937e-05, "loss": 0.0023948252201080322, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.0024, "step": 227, "tokens/total": 7347336, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 103219} +{"epoch": 0.890625, "grad_norm": 0.04663617163896561, "learning_rate": 6.69036847577983e-05, "loss": 0.00031277682865038514, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00031, "step": 228, "tokens/total": 7377592, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 103675} +{"epoch": 0.89453125, "grad_norm": 0.06009799987077713, "learning_rate": 6.662348872453553e-05, "loss": 0.0003573694557417184, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00036, "step": 229, "tokens/total": 7410184, "tokens/train_per_sec_per_gpu": 10.58, "tokens/trainable": 104136} +{"epoch": 0.8984375, "grad_norm": 0.08916021138429642, "learning_rate": 6.63428089904618e-05, "loss": 0.001012815278954804, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00101, "step": 230, "tokens/total": 7443984, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 104588} +{"epoch": 0.90234375, "grad_norm": 0.29150617122650146, "learning_rate": 6.60616572358065e-05, "loss": 0.0005768929258920252, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00058, "step": 231, "tokens/total": 7472480, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 104988} +{"epoch": 0.90625, "grad_norm": 0.3293347656726837, "learning_rate": 6.578004516044172e-05, "loss": 0.010846973396837711, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.01091, "step": 232, "tokens/total": 7505992, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 105455} +{"epoch": 0.91015625, "grad_norm": 0.03366269916296005, "learning_rate": 6.549798448339548e-05, "loss": 0.0002260541368741542, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00023, "step": 233, "tokens/total": 7541168, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 105918} +{"epoch": 0.9140625, "grad_norm": 0.697310745716095, "learning_rate": 6.521548694236384e-05, "loss": 0.007348157465457916, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00738, "step": 234, "tokens/total": 7574208, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 106355} +{"epoch": 0.91796875, "grad_norm": 0.6252380013465881, "learning_rate": 6.493256429322259e-05, "loss": 0.007455206010490656, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00748, "step": 235, "tokens/total": 7607016, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 106786} +{"epoch": 0.921875, "grad_norm": 1.9974677562713623, "learning_rate": 6.464922830953799e-05, "loss": 0.023213990032672882, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.02349, "step": 236, "tokens/total": 7641744, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 107219} +{"epoch": 0.92578125, "grad_norm": 0.594154953956604, "learning_rate": 6.436549078207688e-05, "loss": 0.023317817598581314, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.02359, "step": 237, "tokens/total": 7673096, "tokens/train_per_sec_per_gpu": 10.22, "tokens/trainable": 107679} +{"epoch": 0.9296875, "grad_norm": 1.0142128467559814, "learning_rate": 6.408136351831592e-05, "loss": 0.015361054800450802, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.01548, "step": 238, "tokens/total": 7705344, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 108119} +{"epoch": 0.93359375, "grad_norm": 0.5048214793205261, "learning_rate": 6.379685834195036e-05, "loss": 0.0032022881787270308, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00321, "step": 239, "tokens/total": 7740240, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 108626} +{"epoch": 0.9375, "grad_norm": 0.4170125126838684, "learning_rate": 6.351198709240186e-05, "loss": 0.004182040691375732, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00419, "step": 240, "tokens/total": 7775392, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 109079} +{"epoch": 0.94140625, "grad_norm": 0.08246399462223053, "learning_rate": 6.32267616243259e-05, "loss": 0.0018019207054749131, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.0018, "step": 241, "tokens/total": 7807296, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 109542} +{"epoch": 0.9453125, "grad_norm": 0.7917425036430359, "learning_rate": 6.294119380711849e-05, "loss": 0.005375212989747524, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00539, "step": 242, "tokens/total": 7838200, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 109969} +{"epoch": 0.94921875, "grad_norm": 0.41765522956848145, "learning_rate": 6.265529552442209e-05, "loss": 0.007667080964893103, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.0077, "step": 243, "tokens/total": 7870936, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 110431} +{"epoch": 0.953125, "grad_norm": 0.13395380973815918, "learning_rate": 6.236907867363127e-05, "loss": 0.0010800037998706102, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00108, "step": 244, "tokens/total": 7902512, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 110877} +{"epoch": 0.95703125, "grad_norm": 0.48151275515556335, "learning_rate": 6.208255516539749e-05, "loss": 0.0023539916146546602, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00236, "step": 245, "tokens/total": 7933224, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 111350} +{"epoch": 0.9609375, "grad_norm": 0.36133283376693726, "learning_rate": 6.179573692313344e-05, "loss": 0.003986855503171682, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00399, "step": 246, "tokens/total": 7968456, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 111798} +{"epoch": 0.96484375, "grad_norm": 1.510582447052002, "learning_rate": 6.150863588251694e-05, "loss": 0.009891650639474392, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00994, "step": 247, "tokens/total": 8003592, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 112264} +{"epoch": 0.96875, "grad_norm": 0.5384044051170349, "learning_rate": 6.122126399099419e-05, "loss": 0.0063748471438884735, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.0064, "step": 248, "tokens/total": 8040048, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 112772} +{"epoch": 0.97265625, "grad_norm": 0.23928534984588623, "learning_rate": 6.0933633207282615e-05, "loss": 0.0011579026468098164, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00116, "step": 249, "tokens/total": 8074000, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 113278} +{"epoch": 0.9765625, "grad_norm": 0.11939486861228943, "learning_rate": 6.064575550087316e-05, "loss": 0.0008515830268152058, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00085, "step": 250, "tokens/total": 8108304, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 113716} +{"epoch": 0.98046875, "grad_norm": 0.1181306391954422, "learning_rate": 6.0357642851532245e-05, "loss": 0.0008698446908965707, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00087, "step": 251, "tokens/total": 8143464, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 114207} +{"epoch": 0.984375, "grad_norm": 0.19507302343845367, "learning_rate": 6.0069307248803294e-05, "loss": 0.00398872047662735, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.004, "step": 252, "tokens/total": 8175576, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 114628} +{"epoch": 0.98828125, "grad_norm": 0.009173001162707806, "learning_rate": 5.9780760691507635e-05, "loss": 0.0001451723655918613, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00015, "step": 253, "tokens/total": 8205528, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 115060} +{"epoch": 0.9921875, "grad_norm": 0.013532533310353756, "learning_rate": 5.9492015187245334e-05, "loss": 0.0001985432900255546, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.0002, "step": 254, "tokens/total": 8239424, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 115514} +{"epoch": 0.99609375, "grad_norm": 0.9770844578742981, "learning_rate": 5.920308275189541e-05, "loss": 0.002238090615719557, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00224, "step": 255, "tokens/total": 8270392, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 115915} +{"epoch": 1.0, "grad_norm": 0.2450684756040573, "learning_rate": 5.8913975409115874e-05, "loss": 0.001273187343031168, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00127, "step": 256, "tokens/total": 8302096, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 116378} +{"epoch": 1.00390625, "grad_norm": 0.5780816674232483, "learning_rate": 5.8624705189843395e-05, "loss": 0.004770210478454828, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00478, "step": 257, "tokens/total": 8337064, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 116865} +{"epoch": 1.0078125, "grad_norm": 1.1035937070846558, "learning_rate": 5.833528413179249e-05, "loss": 0.008804233744740486, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00884, "step": 258, "tokens/total": 8372840, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 117363} +{"epoch": 1.01171875, "grad_norm": 0.8923096060752869, "learning_rate": 5.80457242789548e-05, "loss": 0.012778133153915405, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.01286, "step": 259, "tokens/total": 8409728, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 117860} +{"epoch": 1.015625, "grad_norm": 0.02320251613855362, "learning_rate": 5.77560376810977e-05, "loss": 0.00028618099167943, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.00029, "step": 260, "tokens/total": 8442296, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 118324} +{"epoch": 1.01953125, "grad_norm": 0.05295966938138008, "learning_rate": 5.7466236393263005e-05, "loss": 0.000789005309343338, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00079, "step": 261, "tokens/total": 8474568, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 118770} +{"epoch": 1.0234375, "grad_norm": 0.43424543738365173, "learning_rate": 5.717633247526522e-05, "loss": 0.0032052008900791407, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00321, "step": 262, "tokens/total": 8505520, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 119206} +{"epoch": 1.02734375, "grad_norm": 0.24586345255374908, "learning_rate": 5.688633799118971e-05, "loss": 0.002974711125716567, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00298, "step": 263, "tokens/total": 8535064, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 119610} +{"epoch": 1.03125, "grad_norm": 0.017072275280952454, "learning_rate": 5.659626500889066e-05, "loss": 0.00020562013378366828, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00021, "step": 264, "tokens/total": 8563600, "tokens/train_per_sec_per_gpu": 7.09, "tokens/trainable": 120080} +{"epoch": 1.03515625, "grad_norm": 7.061514854431152, "learning_rate": 5.6306125599488905e-05, "loss": 0.0181676484644413, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.01833, "step": 265, "tokens/total": 8592648, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 120519} +{"epoch": 1.0390625, "grad_norm": 0.3923277258872986, "learning_rate": 5.601593183686955e-05, "loss": 0.0033699069172143936, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00338, "step": 266, "tokens/total": 8627328, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 120995} +{"epoch": 1.04296875, "grad_norm": 1.251725196838379, "learning_rate": 5.572569579717961e-05, "loss": 0.014813398942351341, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.01492, "step": 267, "tokens/total": 8659760, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 121459} +{"epoch": 1.046875, "grad_norm": 0.1546374410390854, "learning_rate": 5.543542955832538e-05, "loss": 0.0012787869200110435, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00128, "step": 268, "tokens/total": 8696968, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 121902} +{"epoch": 1.05078125, "grad_norm": 0.5323899984359741, "learning_rate": 5.514514519946986e-05, "loss": 0.00999853853136301, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01005, "step": 269, "tokens/total": 8735392, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 122399} +{"epoch": 1.0546875, "grad_norm": 0.04301890358328819, "learning_rate": 5.485485480053015e-05, "loss": 0.0005380926886573434, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00054, "step": 270, "tokens/total": 8769840, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 122804} +{"epoch": 1.05859375, "grad_norm": 0.07786845415830612, "learning_rate": 5.4564570441674645e-05, "loss": 0.0010392982512712479, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00104, "step": 271, "tokens/total": 8800648, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 123244} +{"epoch": 1.0625, "grad_norm": 0.17377372086048126, "learning_rate": 5.42743042028204e-05, "loss": 0.0011218319414183497, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00112, "step": 272, "tokens/total": 8832480, "tokens/train_per_sec_per_gpu": 6.84, "tokens/trainable": 123683} +{"epoch": 1.06640625, "grad_norm": 0.05215458944439888, "learning_rate": 5.3984068163130464e-05, "loss": 0.0007553771138191223, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00076, "step": 273, "tokens/total": 8864840, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 124153} +{"epoch": 1.0703125, "grad_norm": 0.33478185534477234, "learning_rate": 5.369387440051111e-05, "loss": 0.002826196374371648, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00283, "step": 274, "tokens/total": 8897272, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 124616} +{"epoch": 1.07421875, "grad_norm": 0.3995516002178192, "learning_rate": 5.340373499110935e-05, "loss": 0.002253021579235792, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00226, "step": 275, "tokens/total": 8930688, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 125109} +{"epoch": 1.078125, "grad_norm": 0.0925586149096489, "learning_rate": 5.3113662008810304e-05, "loss": 0.00032150381593964994, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00032, "step": 276, "tokens/total": 8967216, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 125591} +{"epoch": 1.08203125, "grad_norm": 0.33165243268013, "learning_rate": 5.282366752473479e-05, "loss": 0.0018337555229663849, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00184, "step": 277, "tokens/total": 9001368, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 126065} +{"epoch": 1.0859375, "grad_norm": 0.08424188941717148, "learning_rate": 5.2533763606737005e-05, "loss": 0.0005950459744781256, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.0006, "step": 278, "tokens/total": 9036808, "tokens/train_per_sec_per_gpu": 6.94, "tokens/trainable": 126480} +{"epoch": 1.08984375, "grad_norm": 1.6073813438415527, "learning_rate": 5.224396231890232e-05, "loss": 0.0016970105934888124, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.0017, "step": 279, "tokens/total": 9067840, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 126929} +{"epoch": 1.09375, "grad_norm": 0.1635570228099823, "learning_rate": 5.195427572104522e-05, "loss": 0.0016469176625832915, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00165, "step": 280, "tokens/total": 9098936, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 127366} +{"epoch": 1.09765625, "grad_norm": 0.7316864132881165, "learning_rate": 5.166471586820751e-05, "loss": 0.008831718936562538, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00887, "step": 281, "tokens/total": 9133264, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 127831} +{"epoch": 1.1015625, "grad_norm": 0.05642508342862129, "learning_rate": 5.1375294810156615e-05, "loss": 0.000530218007043004, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00053, "step": 282, "tokens/total": 9165008, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 128278} +{"epoch": 1.10546875, "grad_norm": 0.046424806118011475, "learning_rate": 5.1086024590884144e-05, "loss": 0.0006023236201144755, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.0006, "step": 283, "tokens/total": 9197952, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 128727} +{"epoch": 1.109375, "grad_norm": 0.6475871801376343, "learning_rate": 5.079691724810461e-05, "loss": 0.005681783426553011, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.0057, "step": 284, "tokens/total": 9232576, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 129162} +{"epoch": 1.11328125, "grad_norm": 0.034450944513082504, "learning_rate": 5.0507984812754684e-05, "loss": 0.0003829827473964542, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00038, "step": 285, "tokens/total": 9264568, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 129659} +{"epoch": 1.1171875, "grad_norm": 0.019626187160611153, "learning_rate": 5.021923930849237e-05, "loss": 0.00033152365358546376, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00033, "step": 286, "tokens/total": 9296544, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 130145} +{"epoch": 1.12109375, "grad_norm": 0.052348002791404724, "learning_rate": 4.99306927511967e-05, "loss": 0.0006904045003466308, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00069, "step": 287, "tokens/total": 9327936, "tokens/train_per_sec_per_gpu": 10.51, "tokens/trainable": 130585} +{"epoch": 1.125, "grad_norm": 0.5757617354393005, "learning_rate": 4.964235714846775e-05, "loss": 0.0030301311053335667, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00303, "step": 288, "tokens/total": 9357984, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 131005} +{"epoch": 1.12890625, "grad_norm": 0.05109189450740814, "learning_rate": 4.9354244499126866e-05, "loss": 0.0004811554681509733, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00048, "step": 289, "tokens/total": 9388664, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 131471} +{"epoch": 1.1328125, "grad_norm": 0.2122468799352646, "learning_rate": 4.90663667927174e-05, "loss": 0.001537843025289476, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00154, "step": 290, "tokens/total": 9420504, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 131920} +{"epoch": 1.13671875, "grad_norm": 0.07476131618022919, "learning_rate": 4.877873600900581e-05, "loss": 0.0006038081482984126, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.0006, "step": 291, "tokens/total": 9452664, "tokens/train_per_sec_per_gpu": 6.34, "tokens/trainable": 132379} +{"epoch": 1.140625, "grad_norm": 0.005699679255485535, "learning_rate": 4.849136411748306e-05, "loss": 8.62247179611586e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00009, "step": 292, "tokens/total": 9488896, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 132851} +{"epoch": 1.14453125, "grad_norm": 0.40911540389060974, "learning_rate": 4.8204263076866574e-05, "loss": 0.008404267951846123, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00844, "step": 293, "tokens/total": 9522112, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 133351} +{"epoch": 1.1484375, "grad_norm": 0.12325657904148102, "learning_rate": 4.791744483460251e-05, "loss": 0.00022522146173287183, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00023, "step": 294, "tokens/total": 9555632, "tokens/train_per_sec_per_gpu": 6.63, "tokens/trainable": 133784} +{"epoch": 1.15234375, "grad_norm": 0.006892908364534378, "learning_rate": 4.7630921326368736e-05, "loss": 6.096101787989028e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00006, "step": 295, "tokens/total": 9590904, "tokens/train_per_sec_per_gpu": 6.46, "tokens/trainable": 134209} +{"epoch": 1.15625, "grad_norm": 0.03309472277760506, "learning_rate": 4.7344704475577916e-05, "loss": 0.00033775100018829107, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00034, "step": 296, "tokens/total": 9620288, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 134645} +{"epoch": 1.16015625, "grad_norm": 0.029709741473197937, "learning_rate": 4.705880619288153e-05, "loss": 0.00031256157672032714, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00031, "step": 297, "tokens/total": 9651984, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 135047} +{"epoch": 1.1640625, "grad_norm": 0.5759191513061523, "learning_rate": 4.677323837567412e-05, "loss": 0.0026388808619230986, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00264, "step": 298, "tokens/total": 9681368, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 135474} +{"epoch": 1.16796875, "grad_norm": 0.00477481447160244, "learning_rate": 4.6488012907598146e-05, "loss": 3.391175778233446e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00003, "step": 299, "tokens/total": 9712872, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 135948} +{"epoch": 1.171875, "grad_norm": 0.055614229291677475, "learning_rate": 4.620314165804964e-05, "loss": 0.00035722737084142864, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00036, "step": 300, "tokens/total": 9745184, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 136421} +{"epoch": 1.17578125, "grad_norm": 0.03903231769800186, "learning_rate": 4.591863648168407e-05, "loss": 0.0001910634309751913, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00019, "step": 301, "tokens/total": 9778928, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 136886} +{"epoch": 1.1796875, "grad_norm": 0.10863807052373886, "learning_rate": 4.5634509217923135e-05, "loss": 0.0006428276537917554, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00064, "step": 302, "tokens/total": 9810752, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 137321} +{"epoch": 1.18359375, "grad_norm": 0.09797024726867676, "learning_rate": 4.535077169046201e-05, "loss": 0.00024851306807249784, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00025, "step": 303, "tokens/total": 9843672, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 137758} +{"epoch": 1.1875, "grad_norm": 0.006719966419041157, "learning_rate": 4.506743570677743e-05, "loss": 7.043276127660647e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00007, "step": 304, "tokens/total": 9873752, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 138190} +{"epoch": 1.19140625, "grad_norm": 0.061736688017845154, "learning_rate": 4.478451305763618e-05, "loss": 0.00030395606881938875, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.0003, "step": 305, "tokens/total": 9906272, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 138650} +{"epoch": 1.1953125, "grad_norm": 0.0042034960351884365, "learning_rate": 4.450201551660454e-05, "loss": 3.474863842711784e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00003, "step": 306, "tokens/total": 9936296, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 139082} +{"epoch": 1.19921875, "grad_norm": 0.29675909876823425, "learning_rate": 4.4219954839558276e-05, "loss": 0.008084367960691452, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00812, "step": 307, "tokens/total": 9967720, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 139523} +{"epoch": 1.203125, "grad_norm": 0.5443748831748962, "learning_rate": 4.393834276419352e-05, "loss": 0.008384747430682182, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.00842, "step": 308, "tokens/total": 9997176, "tokens/train_per_sec_per_gpu": 10.87, "tokens/trainable": 140010} +{"epoch": 1.20703125, "grad_norm": 0.0037054524291306734, "learning_rate": 4.36571910095382e-05, "loss": 2.591164047771599e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00003, "step": 309, "tokens/total": 10031872, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 140454} +{"epoch": 1.2109375, "grad_norm": 0.0035872934386134148, "learning_rate": 4.337651127546448e-05, "loss": 2.9770810215268284e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00003, "step": 310, "tokens/total": 10065240, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 140905} +{"epoch": 1.21484375, "grad_norm": 0.38882917165756226, "learning_rate": 4.3096315242201736e-05, "loss": 0.00267305807210505, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00268, "step": 311, "tokens/total": 10096640, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 141339} +{"epoch": 1.21875, "grad_norm": 0.12217660993337631, "learning_rate": 4.2816614569850635e-05, "loss": 0.0005129615892656147, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00051, "step": 312, "tokens/total": 10130192, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 141778} +{"epoch": 1.22265625, "grad_norm": 0.7974177598953247, "learning_rate": 4.2537420897897864e-05, "loss": 0.012486078776419163, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01256, "step": 313, "tokens/total": 10164248, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 142255} +{"epoch": 1.2265625, "grad_norm": 0.008416273631155491, "learning_rate": 4.225874584473174e-05, "loss": 6.453386595239863e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00006, "step": 314, "tokens/total": 10194664, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 142693} +{"epoch": 1.23046875, "grad_norm": 0.01573019288480282, "learning_rate": 4.19806010071587e-05, "loss": 8.140943828038871e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00008, "step": 315, "tokens/total": 10227768, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 143157} +{"epoch": 1.234375, "grad_norm": 0.21594959497451782, "learning_rate": 4.170299795992081e-05, "loss": 0.0011683881748467684, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00117, "step": 316, "tokens/total": 10260464, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 143603} +{"epoch": 1.23828125, "grad_norm": 0.22568118572235107, "learning_rate": 4.142594825521398e-05, "loss": 0.0014486180152744055, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00145, "step": 317, "tokens/total": 10297528, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 144084} +{"epoch": 1.2421875, "grad_norm": 0.6898549795150757, "learning_rate": 4.114946342220728e-05, "loss": 0.014587861485779285, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01469, "step": 318, "tokens/total": 10331448, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 144532} +{"epoch": 1.24609375, "grad_norm": 0.015151974745094776, "learning_rate": 4.087355496656321e-05, "loss": 0.00019888556562364101, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.0002, "step": 319, "tokens/total": 10363152, "tokens/train_per_sec_per_gpu": 10.22, "tokens/trainable": 145004} +{"epoch": 1.25, "grad_norm": 0.1525793820619583, "learning_rate": 4.05982343699588e-05, "loss": 0.0004204195865895599, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00042, "step": 320, "tokens/total": 10397832, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 145498} +{"epoch": 1.25390625, "grad_norm": 0.1328331083059311, "learning_rate": 4.0323513089607876e-05, "loss": 0.0014618299901485443, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00146, "step": 321, "tokens/total": 10431416, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 145965} +{"epoch": 1.2578125, "grad_norm": 0.27279025316238403, "learning_rate": 4.004940255778431e-05, "loss": 0.0016563832759857178, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00166, "step": 322, "tokens/total": 10467384, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 146409} +{"epoch": 1.26171875, "grad_norm": 0.14881695806980133, "learning_rate": 3.977591418134619e-05, "loss": 0.0016533946618437767, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00165, "step": 323, "tokens/total": 10500792, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 146849} +{"epoch": 1.265625, "grad_norm": 0.03304417058825493, "learning_rate": 3.95030593412612e-05, "loss": 0.00019466623780317605, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00019, "step": 324, "tokens/total": 10532592, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 147272} +{"epoch": 1.26953125, "grad_norm": 0.12991800904273987, "learning_rate": 3.923084939213296e-05, "loss": 0.0015193780418485403, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00152, "step": 325, "tokens/total": 10562392, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 147694} +{"epoch": 1.2734375, "grad_norm": 0.007839553989470005, "learning_rate": 3.895929566172861e-05, "loss": 0.0001354961423203349, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00014, "step": 326, "tokens/total": 10595416, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 148158} +{"epoch": 1.27734375, "grad_norm": 0.05272425338625908, "learning_rate": 3.868840945050728e-05, "loss": 0.0004349810187704861, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00044, "step": 327, "tokens/total": 10630504, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 148590} +{"epoch": 1.28125, "grad_norm": 0.016601964831352234, "learning_rate": 3.841820203114995e-05, "loss": 0.00011955359514104202, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00012, "step": 328, "tokens/total": 10661632, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 149037} +{"epoch": 1.28515625, "grad_norm": 0.009864515624940395, "learning_rate": 3.814868464809027e-05, "loss": 0.0002010842290474102, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.0002, "step": 329, "tokens/total": 10691544, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 149439} +{"epoch": 1.2890625, "grad_norm": 0.18597562611103058, "learning_rate": 3.787986851704667e-05, "loss": 0.0011520149419084191, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00115, "step": 330, "tokens/total": 10724768, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 149869} +{"epoch": 1.29296875, "grad_norm": 0.020407341420650482, "learning_rate": 3.7611764824555654e-05, "loss": 0.00014565022138413042, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00015, "step": 331, "tokens/total": 10756008, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 150343} +{"epoch": 1.296875, "grad_norm": 0.09921498596668243, "learning_rate": 3.734438472750619e-05, "loss": 0.0008085445151664317, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00081, "step": 332, "tokens/total": 10793152, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 150814} +{"epoch": 1.30078125, "grad_norm": 0.18177562952041626, "learning_rate": 3.707773935267552e-05, "loss": 0.001229858840815723, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00123, "step": 333, "tokens/total": 10827624, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 151300} +{"epoch": 1.3046875, "grad_norm": 0.01989387907087803, "learning_rate": 3.68118397962661e-05, "loss": 0.00022922444622963667, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00023, "step": 334, "tokens/total": 10859712, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 151708} +{"epoch": 1.30859375, "grad_norm": 0.02536221221089363, "learning_rate": 3.654669712344384e-05, "loss": 0.0003126200754195452, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00031, "step": 335, "tokens/total": 10888792, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 152164} +{"epoch": 1.3125, "grad_norm": 0.0458482950925827, "learning_rate": 3.628232236787763e-05, "loss": 0.00034477957524359226, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00034, "step": 336, "tokens/total": 10922968, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 152603} +{"epoch": 1.31640625, "grad_norm": 0.07431786507368088, "learning_rate": 3.6018726531280144e-05, "loss": 0.00042704664519988, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00043, "step": 337, "tokens/total": 10958000, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 153094} +{"epoch": 1.3203125, "grad_norm": 0.2314058393239975, "learning_rate": 3.575592058295017e-05, "loss": 0.0028814608231186867, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00289, "step": 338, "tokens/total": 10989872, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 153573} +{"epoch": 1.32421875, "grad_norm": 0.0008781155338510871, "learning_rate": 3.549391545931585e-05, "loss": 1.701676592347212e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00002, "step": 339, "tokens/total": 11020784, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 154033} +{"epoch": 1.328125, "grad_norm": 0.2190881073474884, "learning_rate": 3.5232722063479914e-05, "loss": 0.0013971286825835705, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.0014, "step": 340, "tokens/total": 11053240, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 154482} +{"epoch": 1.33203125, "grad_norm": 0.04455112665891647, "learning_rate": 3.49723512647657e-05, "loss": 0.0003303556004539132, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00033, "step": 341, "tokens/total": 11089040, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 154982} +{"epoch": 1.3359375, "grad_norm": 0.015334265306591988, "learning_rate": 3.471281389826491e-05, "loss": 0.0001342726027360186, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00013, "step": 342, "tokens/total": 11119672, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 155387} +{"epoch": 1.33984375, "grad_norm": 0.03774840757250786, "learning_rate": 3.4454120764386764e-05, "loss": 0.00024028134066611528, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00024, "step": 343, "tokens/total": 11155040, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 155842} +{"epoch": 1.34375, "grad_norm": 0.014823688194155693, "learning_rate": 3.4196282628408526e-05, "loss": 0.00018696454935707152, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00019, "step": 344, "tokens/total": 11185952, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 156287} +{"epoch": 1.34765625, "grad_norm": 0.10823217034339905, "learning_rate": 3.3939310220027456e-05, "loss": 0.0006391166243702173, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00064, "step": 345, "tokens/total": 11219272, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 156782} +{"epoch": 1.3515625, "grad_norm": 0.002855873666703701, "learning_rate": 3.3683214232914404e-05, "loss": 2.9794537113048136e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00003, "step": 346, "tokens/total": 11253424, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 157255} +{"epoch": 1.35546875, "grad_norm": 0.06455519795417786, "learning_rate": 3.342800532426873e-05, "loss": 0.0005804813117720187, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00058, "step": 347, "tokens/total": 11286216, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 157721} +{"epoch": 1.359375, "grad_norm": 0.00937845278531313, "learning_rate": 3.317369411437484e-05, "loss": 0.00015670634456910193, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00016, "step": 348, "tokens/total": 11319480, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 158189} +{"epoch": 1.36328125, "grad_norm": 0.01152450405061245, "learning_rate": 3.292029118616024e-05, "loss": 0.00014356027531903237, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00014, "step": 349, "tokens/total": 11350472, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 158612} +{"epoch": 1.3671875, "grad_norm": 0.051517803221940994, "learning_rate": 3.266780708475511e-05, "loss": 0.0003877044946420938, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00039, "step": 350, "tokens/total": 11383088, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 159090} +{"epoch": 1.37109375, "grad_norm": 0.021377189084887505, "learning_rate": 3.241625231705354e-05, "loss": 0.00010781685705296695, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00011, "step": 351, "tokens/total": 11418392, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 159549} +{"epoch": 1.375, "grad_norm": 0.02083928883075714, "learning_rate": 3.216563735127618e-05, "loss": 4.264710150891915e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00004, "step": 352, "tokens/total": 11449784, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 160026} +{"epoch": 1.37890625, "grad_norm": 0.013635780662298203, "learning_rate": 3.191597261653475e-05, "loss": 9.473154204897583e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00009, "step": 353, "tokens/total": 11484240, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 160487} +{"epoch": 1.3828125, "grad_norm": 0.0031274070497602224, "learning_rate": 3.166726850239794e-05, "loss": 3.8173799111973494e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00004, "step": 354, "tokens/total": 11513360, "tokens/train_per_sec_per_gpu": 11.09, "tokens/trainable": 160935} +{"epoch": 1.38671875, "grad_norm": 0.26715531945228577, "learning_rate": 3.141953535845912e-05, "loss": 0.0014820604119449854, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00148, "step": 355, "tokens/total": 11543112, "tokens/train_per_sec_per_gpu": 10.3, "tokens/trainable": 161385} +{"epoch": 1.390625, "grad_norm": 0.011197442188858986, "learning_rate": 3.11727834939056e-05, "loss": 0.00013717291585635394, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00014, "step": 356, "tokens/total": 11573464, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 161849} +{"epoch": 1.39453125, "grad_norm": 0.06569802016019821, "learning_rate": 3.092702317708967e-05, "loss": 0.0008324727532453835, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00083, "step": 357, "tokens/total": 11604896, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 162306} +{"epoch": 1.3984375, "grad_norm": 0.011035638861358166, "learning_rate": 3.0682264635101276e-05, "loss": 6.825919263064861e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00007, "step": 358, "tokens/total": 11636368, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 162759} +{"epoch": 1.40234375, "grad_norm": 0.1091134175658226, "learning_rate": 3.0438518053342407e-05, "loss": 0.00015353334310930222, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00015, "step": 359, "tokens/total": 11670216, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 163230} +{"epoch": 1.40625, "grad_norm": 0.2502235472202301, "learning_rate": 3.0195793575103266e-05, "loss": 0.0022095313761383295, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00221, "step": 360, "tokens/total": 11707816, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 163694} +{"epoch": 1.41015625, "grad_norm": 0.04895743727684021, "learning_rate": 2.9954101301140146e-05, "loss": 0.00037207311834208667, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00037, "step": 361, "tokens/total": 11742584, "tokens/train_per_sec_per_gpu": 6.96, "tokens/trainable": 164144} +{"epoch": 1.4140625, "grad_norm": 0.011038385331630707, "learning_rate": 2.9713451289255123e-05, "loss": 0.00011145838652737439, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00011, "step": 362, "tokens/total": 11770384, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 164585} +{"epoch": 1.41796875, "grad_norm": 0.013131794519722462, "learning_rate": 2.9473853553877484e-05, "loss": 0.00012399123806972057, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00012, "step": 363, "tokens/total": 11803208, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 165059} +{"epoch": 1.421875, "grad_norm": 3.87949275970459, "learning_rate": 2.9235318065647e-05, "loss": 0.001334474072791636, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00134, "step": 364, "tokens/total": 11840480, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 165519} +{"epoch": 1.42578125, "grad_norm": 0.016668125987052917, "learning_rate": 2.8997854750998964e-05, "loss": 0.00015702139353379607, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00016, "step": 365, "tokens/total": 11875544, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 165984} +{"epoch": 1.4296875, "grad_norm": 0.06344672292470932, "learning_rate": 2.8761473491751258e-05, "loss": 0.00020288745872676373, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.0002, "step": 366, "tokens/total": 11907656, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 166419} +{"epoch": 1.43359375, "grad_norm": 0.007155115716159344, "learning_rate": 2.8526184124692883e-05, "loss": 3.500051025184803e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00004, "step": 367, "tokens/total": 11941904, "tokens/train_per_sec_per_gpu": 6.43, "tokens/trainable": 166857} +{"epoch": 1.4375, "grad_norm": 0.021127179265022278, "learning_rate": 2.829199644117484e-05, "loss": 8.050489122979343e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00008, "step": 368, "tokens/total": 11973624, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 167267} +{"epoch": 1.44140625, "grad_norm": 0.0870605930685997, "learning_rate": 2.8058920186702553e-05, "loss": 0.0001545553095638752, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00015, "step": 369, "tokens/total": 12010360, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 167740} +{"epoch": 1.4453125, "grad_norm": 0.0014929031021893024, "learning_rate": 2.782696506053033e-05, "loss": 1.8325908968108706e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00002, "step": 370, "tokens/total": 12044544, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 168172} +{"epoch": 1.44921875, "grad_norm": 0.039372529834508896, "learning_rate": 2.7596140715257824e-05, "loss": 0.00025936932070180774, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00026, "step": 371, "tokens/total": 12075792, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 168637} +{"epoch": 1.453125, "grad_norm": 0.027524098753929138, "learning_rate": 2.7366456756428184e-05, "loss": 0.00018690136494114995, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00019, "step": 372, "tokens/total": 12108088, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 169094} +{"epoch": 1.45703125, "grad_norm": 0.24282237887382507, "learning_rate": 2.7137922742128486e-05, "loss": 0.0007423263159580529, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00074, "step": 373, "tokens/total": 12140888, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 169593} +{"epoch": 1.4609375, "grad_norm": 0.1707821637392044, "learning_rate": 2.691054818259188e-05, "loss": 0.0011599954450502992, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00116, "step": 374, "tokens/total": 12174560, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 170066} +{"epoch": 1.46484375, "grad_norm": 0.0014547965256497264, "learning_rate": 2.6684342539801933e-05, "loss": 1.9985376638942398e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00002, "step": 375, "tokens/total": 12204848, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 170518} +{"epoch": 1.46875, "grad_norm": 0.02336573600769043, "learning_rate": 2.645931522709877e-05, "loss": 0.00014505830768030137, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00015, "step": 376, "tokens/total": 12238472, "tokens/train_per_sec_per_gpu": 10.84, "tokens/trainable": 170998} +{"epoch": 1.47265625, "grad_norm": 0.0323311984539032, "learning_rate": 2.6235475608787365e-05, "loss": 0.00024221389321610332, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00024, "step": 377, "tokens/total": 12271024, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 171459} +{"epoch": 1.4765625, "grad_norm": 0.003342488780617714, "learning_rate": 2.6012832999747916e-05, "loss": 6.0761165514122695e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00006, "step": 378, "tokens/total": 12301312, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 171905} +{"epoch": 1.48046875, "grad_norm": 1.6876935958862305, "learning_rate": 2.579139666504821e-05, "loss": 0.008571377955377102, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00861, "step": 379, "tokens/total": 12337072, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 172351} +{"epoch": 1.484375, "grad_norm": 0.009042182937264442, "learning_rate": 2.557117581955798e-05, "loss": 4.289774369681254e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00004, "step": 380, "tokens/total": 12368000, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 172817} +{"epoch": 1.48828125, "grad_norm": 0.009125811979174614, "learning_rate": 2.5352179627565532e-05, "loss": 4.790291131939739e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00005, "step": 381, "tokens/total": 12399880, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 173258} +{"epoch": 1.4921875, "grad_norm": 0.36612212657928467, "learning_rate": 2.5134417202396277e-05, "loss": 0.0013663998106494546, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00137, "step": 382, "tokens/total": 12432288, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 173665} +{"epoch": 1.49609375, "grad_norm": 0.06363425403833389, "learning_rate": 2.491789760603361e-05, "loss": 0.0002047839661827311, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.0002, "step": 383, "tokens/total": 12465592, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 174095} +{"epoch": 1.5, "grad_norm": 0.06855729967355728, "learning_rate": 2.4702629848741764e-05, "loss": 0.00029772185371257365, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.0003, "step": 384, "tokens/total": 12498200, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 174550} +{"epoch": 1.50390625, "grad_norm": 0.006325738504528999, "learning_rate": 2.4488622888690785e-05, "loss": 6.635349564021453e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00007, "step": 385, "tokens/total": 12530280, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 174990} +{"epoch": 1.5078125, "grad_norm": 0.0032503672409802675, "learning_rate": 2.427588563158384e-05, "loss": 2.4833512725308537e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00002, "step": 386, "tokens/total": 12563360, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 175476} +{"epoch": 1.51171875, "grad_norm": 0.19394826889038086, "learning_rate": 2.406442693028651e-05, "loss": 0.0008008039440028369, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.0008, "step": 387, "tokens/total": 12593032, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 175897} +{"epoch": 1.515625, "grad_norm": 0.038925886154174805, "learning_rate": 2.3854255584458547e-05, "loss": 4.2194056732114404e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00004, "step": 388, "tokens/total": 12628272, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 176324} +{"epoch": 1.51953125, "grad_norm": 0.0008955897064879537, "learning_rate": 2.3645380340187508e-05, "loss": 1.5502017049584538e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00002, "step": 389, "tokens/total": 12661368, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 176805} +{"epoch": 1.5234375, "grad_norm": 0.13275980949401855, "learning_rate": 2.3437809889624914e-05, "loss": 0.0009544279309920967, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.77, "memory/max_allocated (GiB)": 63.77, "ppl": 1.00095, "step": 390, "tokens/total": 12691168, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 177256} +{"epoch": 1.52734375, "grad_norm": 0.0013756570406258106, "learning_rate": 2.3231552870624487e-05, "loss": 1.9104330931440927e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00002, "step": 391, "tokens/total": 12721264, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 177682} +{"epoch": 1.53125, "grad_norm": 0.018446210771799088, "learning_rate": 2.3026617866382657e-05, "loss": 0.00010704064334277064, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00011, "step": 392, "tokens/total": 12753640, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 178179} +{"epoch": 1.53515625, "grad_norm": 0.04184940457344055, "learning_rate": 2.2823013405081507e-05, "loss": 0.00014302060299087316, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00014, "step": 393, "tokens/total": 12787056, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 178676} +{"epoch": 1.5390625, "grad_norm": 0.0009975213324651122, "learning_rate": 2.2620747959533722e-05, "loss": 1.458782389818225e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00001, "step": 394, "tokens/total": 12819312, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 179173} +{"epoch": 1.54296875, "grad_norm": 0.004811645019799471, "learning_rate": 2.2419829946830123e-05, "loss": 3.864563404931687e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00004, "step": 395, "tokens/total": 12853376, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 179624} +{"epoch": 1.546875, "grad_norm": 0.026116183027625084, "learning_rate": 2.2220267727989325e-05, "loss": 0.0001434308651369065, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00014, "step": 396, "tokens/total": 12882960, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 180078} +{"epoch": 1.55078125, "grad_norm": 0.3623203933238983, "learning_rate": 2.202206960760984e-05, "loss": 0.0015740969683974981, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00158, "step": 397, "tokens/total": 12917016, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 180585} +{"epoch": 1.5546875, "grad_norm": 0.002089948859065771, "learning_rate": 2.182524383352446e-05, "loss": 1.5697660273872316e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00002, "step": 398, "tokens/total": 12950056, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 181039} +{"epoch": 1.55859375, "grad_norm": 0.1087174043059349, "learning_rate": 2.1629798596457056e-05, "loss": 0.0006332685588859022, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00063, "step": 399, "tokens/total": 12981728, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 181489} +{"epoch": 1.5625, "grad_norm": 0.33820420503616333, "learning_rate": 2.1435742029681725e-05, "loss": 0.0015021745348349214, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.0015, "step": 400, "tokens/total": 13011464, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 181906} +{"epoch": 1.56640625, "grad_norm": 0.008120374754071236, "learning_rate": 2.124308220868431e-05, "loss": 8.189879008568823e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00008, "step": 401, "tokens/total": 13044432, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 182361} +{"epoch": 1.5703125, "grad_norm": 0.01710631139576435, "learning_rate": 2.105182715082638e-05, "loss": 8.941714622778818e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00009, "step": 402, "tokens/total": 13075608, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 182838} +{"epoch": 1.57421875, "grad_norm": 0.0008950813207775354, "learning_rate": 2.0861984815011552e-05, "loss": 7.533315510954708e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00001, "step": 403, "tokens/total": 13107352, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 183269} +{"epoch": 1.578125, "grad_norm": 0.002938683610409498, "learning_rate": 2.0673563101354323e-05, "loss": 2.779942224151455e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00003, "step": 404, "tokens/total": 13140736, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 183699} +{"epoch": 1.58203125, "grad_norm": 0.2819118797779083, "learning_rate": 2.0486569850851317e-05, "loss": 0.010766681283712387, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.01082, "step": 405, "tokens/total": 13174040, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 184162} +{"epoch": 1.5859375, "grad_norm": 0.0014408738352358341, "learning_rate": 2.0301012845054956e-05, "loss": 2.1799158275825903e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00002, "step": 406, "tokens/total": 13207232, "tokens/train_per_sec_per_gpu": 6.65, "tokens/trainable": 184576} +{"epoch": 1.58984375, "grad_norm": 0.058856308460235596, "learning_rate": 2.011689980574966e-05, "loss": 0.00025181230739690363, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00025, "step": 407, "tokens/total": 13241160, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 184995} +{"epoch": 1.59375, "grad_norm": 0.0021246036048978567, "learning_rate": 1.993423839463052e-05, "loss": 2.0188017515465617e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00002, "step": 408, "tokens/total": 13273104, "tokens/train_per_sec_per_gpu": 10.5, "tokens/trainable": 185452} +{"epoch": 1.59765625, "grad_norm": 0.011141506023705006, "learning_rate": 1.975303621298445e-05, "loss": 4.311702286940999e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00004, "step": 409, "tokens/total": 13308136, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 185904} +{"epoch": 1.6015625, "grad_norm": 0.0003598052717279643, "learning_rate": 1.957330080137385e-05, "loss": 6.976109943934716e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00001, "step": 410, "tokens/total": 13337816, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 186355} +{"epoch": 1.60546875, "grad_norm": 0.012044641189277172, "learning_rate": 1.9395039639322864e-05, "loss": 8.8953209342435e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00009, "step": 411, "tokens/total": 13372576, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 186831} +{"epoch": 1.609375, "grad_norm": 0.0015376565279439092, "learning_rate": 1.9218260145006073e-05, "loss": 6.043358098395402e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00001, "step": 412, "tokens/total": 13406664, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 187317} +{"epoch": 1.61328125, "grad_norm": 0.0004661590210162103, "learning_rate": 1.904296967493982e-05, "loss": 8.641689419164322e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00001, "step": 413, "tokens/total": 13438256, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 187773} +{"epoch": 1.6171875, "grad_norm": 0.0060728988610208035, "learning_rate": 1.8869175523676064e-05, "loss": 4.426252417033538e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00004, "step": 414, "tokens/total": 13472096, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 188268} +{"epoch": 1.62109375, "grad_norm": 0.0012627039104700089, "learning_rate": 1.869688492349885e-05, "loss": 1.7591193682164885e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00002, "step": 415, "tokens/total": 13504192, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 188720} +{"epoch": 1.625, "grad_norm": 0.002823289716616273, "learning_rate": 1.85261050441233e-05, "loss": 3.750734686036594e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00004, "step": 416, "tokens/total": 13537392, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 189177} +{"epoch": 1.62890625, "grad_norm": 0.27001747488975525, "learning_rate": 1.8356842992397304e-05, "loss": 0.001587031176313758, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00159, "step": 417, "tokens/total": 13569344, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 189617} +{"epoch": 1.6328125, "grad_norm": 0.2196461409330368, "learning_rate": 1.8189105812005714e-05, "loss": 0.0009314907365478575, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00093, "step": 418, "tokens/total": 13602200, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 190028} +{"epoch": 1.63671875, "grad_norm": 0.004646007437258959, "learning_rate": 1.802290048317732e-05, "loss": 2.2323667508317158e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00002, "step": 419, "tokens/total": 13632640, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 190445} +{"epoch": 1.640625, "grad_norm": 0.007928989827632904, "learning_rate": 1.785823392239424e-05, "loss": 1.4002576790517196e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00001, "step": 420, "tokens/total": 13664920, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 190886} +{"epoch": 1.64453125, "grad_norm": 0.010526476427912712, "learning_rate": 1.7695112982104225e-05, "loss": 3.607779581216164e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00004, "step": 421, "tokens/total": 13697144, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 191325} +{"epoch": 1.6484375, "grad_norm": 0.44544747471809387, "learning_rate": 1.7533544450435433e-05, "loss": 0.002855646889656782, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00286, "step": 422, "tokens/total": 13724496, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 191758} +{"epoch": 1.65234375, "grad_norm": 0.00031105350353755057, "learning_rate": 1.7373535050913946e-05, "loss": 5.9802464420499746e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00001, "step": 423, "tokens/total": 13757880, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 192195} +{"epoch": 1.65625, "grad_norm": 0.05882161110639572, "learning_rate": 1.721509144218405e-05, "loss": 0.00031213462352752686, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00031, "step": 424, "tokens/total": 13787648, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 192625} +{"epoch": 1.66015625, "grad_norm": 0.0008937187376432121, "learning_rate": 1.705822021773101e-05, "loss": 1.024833272822434e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00001, "step": 425, "tokens/total": 13819720, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 193111} +{"epoch": 1.6640625, "grad_norm": 0.014127750881016254, "learning_rate": 1.69029279056068e-05, "loss": 0.0001027590042212978, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.0001, "step": 426, "tokens/total": 13854944, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 193551} +{"epoch": 1.66796875, "grad_norm": 0.24893109500408173, "learning_rate": 1.6749220968158415e-05, "loss": 0.005561263300478458, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00558, "step": 427, "tokens/total": 13887824, "tokens/train_per_sec_per_gpu": 10.43, "tokens/trainable": 194057} +{"epoch": 1.671875, "grad_norm": 0.004814503248780966, "learning_rate": 1.659710580175893e-05, "loss": 5.2404684538487345e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00005, "step": 428, "tokens/total": 13918192, "tokens/train_per_sec_per_gpu": 10.2, "tokens/trainable": 194556} +{"epoch": 1.67578125, "grad_norm": 0.0020030499435961246, "learning_rate": 1.644658873654133e-05, "loss": 2.372681046836078e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00002, "step": 429, "tokens/total": 13947440, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 194993} +{"epoch": 1.6796875, "grad_norm": 0.01364092156291008, "learning_rate": 1.629767603613508e-05, "loss": 3.5511831811163574e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00004, "step": 430, "tokens/total": 13974360, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 195393} +{"epoch": 1.68359375, "grad_norm": 0.061539873480796814, "learning_rate": 1.615037389740547e-05, "loss": 0.0003403580340091139, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00034, "step": 431, "tokens/total": 14006592, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 195837} +{"epoch": 1.6875, "grad_norm": 0.03447442129254341, "learning_rate": 1.600468845019576e-05, "loss": 0.00015946311759762466, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00016, "step": 432, "tokens/total": 14040776, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 196317} +{"epoch": 1.69140625, "grad_norm": 0.006206897087395191, "learning_rate": 1.5860625757072092e-05, "loss": 1.2679005521931686e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00001, "step": 433, "tokens/total": 14073040, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 196765} +{"epoch": 1.6953125, "grad_norm": 0.005987922195345163, "learning_rate": 1.571819181307116e-05, "loss": 3.0454026273218915e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00003, "step": 434, "tokens/total": 14107624, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 197237} +{"epoch": 1.69921875, "grad_norm": 0.002897675847634673, "learning_rate": 1.557739254545075e-05, "loss": 3.1634590413887054e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00003, "step": 435, "tokens/total": 14137784, "tokens/train_per_sec_per_gpu": 10.71, "tokens/trainable": 197709} +{"epoch": 1.703125, "grad_norm": 0.0014296910958364606, "learning_rate": 1.543823381344311e-05, "loss": 1.812200207496062e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00002, "step": 436, "tokens/total": 14167776, "tokens/train_per_sec_per_gpu": 10.83, "tokens/trainable": 198184} +{"epoch": 1.70703125, "grad_norm": 0.0037678806111216545, "learning_rate": 1.5300721408011114e-05, "loss": 4.046248795930296e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00004, "step": 437, "tokens/total": 14199752, "tokens/train_per_sec_per_gpu": 10.3, "tokens/trainable": 198641} +{"epoch": 1.7109375, "grad_norm": 0.0009564560023136437, "learning_rate": 1.5164861051607254e-05, "loss": 1.0270579878124408e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.00001, "step": 438, "tokens/total": 14230344, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 199049} +{"epoch": 1.71484375, "grad_norm": 0.00028030495741404593, "learning_rate": 1.5030658397935521e-05, "loss": 6.889945780130802e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00001, "step": 439, "tokens/total": 14263808, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 199518} +{"epoch": 1.71875, "grad_norm": 0.0029570977203547955, "learning_rate": 1.4898119031716104e-05, "loss": 2.75811198662268e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00003, "step": 440, "tokens/total": 14295864, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 199976} +{"epoch": 1.72265625, "grad_norm": 0.7391864657402039, "learning_rate": 1.476724846845306e-05, "loss": 0.011948215775191784, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.01202, "step": 441, "tokens/total": 14331688, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 200453} +{"epoch": 1.7265625, "grad_norm": 0.029471050947904587, "learning_rate": 1.463805215420471e-05, "loss": 0.00012792057532351464, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00013, "step": 442, "tokens/total": 14364688, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 200923} +{"epoch": 1.73046875, "grad_norm": 0.007126951590180397, "learning_rate": 1.451053546535705e-05, "loss": 4.77697431051638e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00005, "step": 443, "tokens/total": 14397832, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 201379} +{"epoch": 1.734375, "grad_norm": 0.05339258164167404, "learning_rate": 1.438470370840001e-05, "loss": 0.00015959674783516675, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00016, "step": 444, "tokens/total": 14431864, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 201851} +{"epoch": 1.73828125, "grad_norm": 0.7859528660774231, "learning_rate": 1.4260562119706606e-05, "loss": 0.005325036123394966, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00534, "step": 445, "tokens/total": 14462152, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 202325} +{"epoch": 1.7421875, "grad_norm": 0.002174032386392355, "learning_rate": 1.413811586531508e-05, "loss": 3.2870790164452046e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00003, "step": 446, "tokens/total": 14493072, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 202766} +{"epoch": 1.74609375, "grad_norm": 0.07933968305587769, "learning_rate": 1.4017370040713884e-05, "loss": 0.00021293206373229623, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00021, "step": 447, "tokens/total": 14527392, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 203215} +{"epoch": 1.75, "grad_norm": 0.020538270473480225, "learning_rate": 1.3898329670629645e-05, "loss": 0.00014061976980883628, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00014, "step": 448, "tokens/total": 14561992, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 203660} +{"epoch": 1.75390625, "grad_norm": 0.0005385893164202571, "learning_rate": 1.3780999708818058e-05, "loss": 7.46101068216376e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00001, "step": 449, "tokens/total": 14592056, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 204110} +{"epoch": 1.7578125, "grad_norm": 0.009593522176146507, "learning_rate": 1.3665385037857758e-05, "loss": 8.308669930556789e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00008, "step": 450, "tokens/total": 14622816, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 204562} +{"epoch": 1.76171875, "grad_norm": 0.03644828498363495, "learning_rate": 1.3551490468947126e-05, "loss": 0.0001887019898276776, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00019, "step": 451, "tokens/total": 14653272, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 205008} +{"epoch": 1.765625, "grad_norm": 0.016611339524388313, "learning_rate": 1.3439320741704075e-05, "loss": 5.004035119782202e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00005, "step": 452, "tokens/total": 14683952, "tokens/train_per_sec_per_gpu": 10.11, "tokens/trainable": 205432} +{"epoch": 1.76953125, "grad_norm": 0.08491484075784683, "learning_rate": 1.3328880523968808e-05, "loss": 0.0004339523729868233, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00043, "step": 453, "tokens/total": 14716256, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 205895} +{"epoch": 1.7734375, "grad_norm": 0.2398563027381897, "learning_rate": 1.3220174411609587e-05, "loss": 0.0029755716677755117, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00298, "step": 454, "tokens/total": 14749088, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 206334} +{"epoch": 1.77734375, "grad_norm": 0.0026082240510731936, "learning_rate": 1.3113206928331471e-05, "loss": 2.2922657080926e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00002, "step": 455, "tokens/total": 14777384, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 206796} +{"epoch": 1.78125, "grad_norm": 0.01891951635479927, "learning_rate": 1.300798252548806e-05, "loss": 0.00017125926387961954, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00017, "step": 456, "tokens/total": 14809640, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 207248} +{"epoch": 1.78515625, "grad_norm": 0.3868103623390198, "learning_rate": 1.2904505581896265e-05, "loss": 0.00632791081443429, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00635, "step": 457, "tokens/total": 14843728, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 207721} +{"epoch": 1.7890625, "grad_norm": 0.011954589746892452, "learning_rate": 1.2802780403654082e-05, "loss": 0.000220512505620718, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00022, "step": 458, "tokens/total": 14876256, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 208168} +{"epoch": 1.79296875, "grad_norm": 0.004615702200680971, "learning_rate": 1.2702811223961408e-05, "loss": 5.615886038867757e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00006, "step": 459, "tokens/total": 14907000, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 208629} +{"epoch": 1.796875, "grad_norm": 0.0014549370389431715, "learning_rate": 1.2604602202943861e-05, "loss": 2.365529871894978e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00002, "step": 460, "tokens/total": 14937704, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 209073} +{"epoch": 1.80078125, "grad_norm": 0.006373634561896324, "learning_rate": 1.2508157427479686e-05, "loss": 8.44542300910689e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00008, "step": 461, "tokens/total": 14966920, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 209520} +{"epoch": 1.8046875, "grad_norm": 0.012350148521363735, "learning_rate": 1.2413480911029655e-05, "loss": 8.527172030881047e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00009, "step": 462, "tokens/total": 14996040, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 209958} +{"epoch": 1.80859375, "grad_norm": 0.04422426223754883, "learning_rate": 1.2320576593470082e-05, "loss": 0.00030532109667547047, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00031, "step": 463, "tokens/total": 15029096, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 210385} +{"epoch": 1.8125, "grad_norm": 0.5227811336517334, "learning_rate": 1.2229448340928828e-05, "loss": 0.0030096229165792465, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00301, "step": 464, "tokens/total": 15063696, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 210893} +{"epoch": 1.81640625, "grad_norm": 0.024008557200431824, "learning_rate": 1.2140099945624458e-05, "loss": 0.00012863072333857417, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00013, "step": 465, "tokens/total": 15098264, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 211350} +{"epoch": 1.8203125, "grad_norm": 0.04980019852519035, "learning_rate": 1.205253512570841e-05, "loss": 0.00035649945493787527, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00036, "step": 466, "tokens/total": 15130464, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 211798} +{"epoch": 1.82421875, "grad_norm": 0.009358668699860573, "learning_rate": 1.1966757525110255e-05, "loss": 0.00011778524640249088, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00012, "step": 467, "tokens/total": 15158736, "tokens/train_per_sec_per_gpu": 10.16, "tokens/trainable": 212249} +{"epoch": 1.828125, "grad_norm": 0.21887321770191193, "learning_rate": 1.1882770713386095e-05, "loss": 0.0015296044293791056, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00153, "step": 468, "tokens/total": 15193648, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 212724} +{"epoch": 1.83203125, "grad_norm": 0.3202848732471466, "learning_rate": 1.180057818556998e-05, "loss": 0.0007014721049927175, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.0007, "step": 469, "tokens/total": 15227240, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 213199} +{"epoch": 1.8359375, "grad_norm": 0.005410423502326012, "learning_rate": 1.1720183362028494e-05, "loss": 5.1420836825855076e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00005, "step": 470, "tokens/total": 15259968, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 213664} +{"epoch": 1.83984375, "grad_norm": 0.15526723861694336, "learning_rate": 1.1641589588318387e-05, "loss": 9.587601380189881e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.0001, "step": 471, "tokens/total": 15292480, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 214109} +{"epoch": 1.84375, "grad_norm": 0.046621665358543396, "learning_rate": 1.1564800135047418e-05, "loss": 0.00010754169488791376, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00011, "step": 472, "tokens/total": 15324976, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 214597} +{"epoch": 1.84765625, "grad_norm": 0.015293599106371403, "learning_rate": 1.148981819773816e-05, "loss": 6.14620148553513e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00006, "step": 473, "tokens/total": 15356448, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 215014} +{"epoch": 1.8515625, "grad_norm": 0.019691357389092445, "learning_rate": 1.1416646896695086e-05, "loss": 0.00013193217455409467, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00013, "step": 474, "tokens/total": 15389192, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 215466} +{"epoch": 1.85546875, "grad_norm": 0.019334876909852028, "learning_rate": 1.1345289276874717e-05, "loss": 0.0002291607524966821, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00023, "step": 475, "tokens/total": 15422728, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 215937} +{"epoch": 1.859375, "grad_norm": 0.07614016532897949, "learning_rate": 1.1275748307758873e-05, "loss": 0.0003094129788223654, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00031, "step": 476, "tokens/total": 15457104, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 216405} +{"epoch": 1.86328125, "grad_norm": 0.028510889038443565, "learning_rate": 1.1208026883231147e-05, "loss": 0.00020276388386264443, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.0002, "step": 477, "tokens/total": 15486592, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 216820} +{"epoch": 1.8671875, "grad_norm": 0.22166873514652252, "learning_rate": 1.1142127821456433e-05, "loss": 0.0010007917881011963, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.001, "step": 478, "tokens/total": 15518368, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 217270} +{"epoch": 1.87109375, "grad_norm": 0.0011973929358646274, "learning_rate": 1.1078053864763674e-05, "loss": 2.0770619812537916e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00002, "step": 479, "tokens/total": 15551256, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 217722} +{"epoch": 1.875, "grad_norm": 0.6300071477890015, "learning_rate": 1.1015807679531756e-05, "loss": 0.0069034467451274395, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00693, "step": 480, "tokens/total": 15581480, "tokens/train_per_sec_per_gpu": 10.04, "tokens/trainable": 218170} +{"epoch": 1.87890625, "grad_norm": 0.0028679263778030872, "learning_rate": 1.0955391856078528e-05, "loss": 3.2174826628761366e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00003, "step": 481, "tokens/total": 15615104, "tokens/train_per_sec_per_gpu": 10.39, "tokens/trainable": 218644} +{"epoch": 1.8828125, "grad_norm": 0.008435295894742012, "learning_rate": 1.0896808908553007e-05, "loss": 9.600802877685055e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.0001, "step": 482, "tokens/total": 15645384, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 219092} +{"epoch": 1.88671875, "grad_norm": 0.012122329324483871, "learning_rate": 1.0840061274830763e-05, "loss": 6.179865158628672e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00006, "step": 483, "tokens/total": 15676600, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 219558} +{"epoch": 1.890625, "grad_norm": 0.001309512765146792, "learning_rate": 1.0785151316412473e-05, "loss": 1.5803409041836858e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00002, "step": 484, "tokens/total": 15709712, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 220007} +{"epoch": 1.89453125, "grad_norm": 0.004899303428828716, "learning_rate": 1.0732081318325639e-05, "loss": 3.9481677958974615e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00004, "step": 485, "tokens/total": 15747848, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 220505} +{"epoch": 1.8984375, "grad_norm": 0.12917064130306244, "learning_rate": 1.0680853489029501e-05, "loss": 0.0003587081446312368, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00036, "step": 486, "tokens/total": 15782416, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 220936} +{"epoch": 1.90234375, "grad_norm": 0.1222563087940216, "learning_rate": 1.0631469960323152e-05, "loss": 0.0005781762301921844, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00058, "step": 487, "tokens/total": 15815456, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 221372} +{"epoch": 1.90625, "grad_norm": 0.03256118297576904, "learning_rate": 1.0583932787256783e-05, "loss": 0.00022520896163769066, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00023, "step": 488, "tokens/total": 15846496, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 221792} +{"epoch": 1.91015625, "grad_norm": 0.001354391104541719, "learning_rate": 1.0538243948046206e-05, "loss": 1.753922515490558e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00002, "step": 489, "tokens/total": 15878784, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 222238} +{"epoch": 1.9140625, "grad_norm": 0.004516978748142719, "learning_rate": 1.0494405343990523e-05, "loss": 3.678444409160875e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00004, "step": 490, "tokens/total": 15909616, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 222682} +{"epoch": 1.91796875, "grad_norm": 0.012694467790424824, "learning_rate": 1.0452418799392985e-05, "loss": 5.679187597706914e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00006, "step": 491, "tokens/total": 15941400, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 223142} +{"epoch": 1.921875, "grad_norm": 0.31908905506134033, "learning_rate": 1.0412286061485102e-05, "loss": 0.0013064577942714095, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00131, "step": 492, "tokens/total": 15975440, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 223609} +{"epoch": 1.92578125, "grad_norm": 0.00895836390554905, "learning_rate": 1.03740088003539e-05, "loss": 3.433962774579413e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00003, "step": 493, "tokens/total": 16004224, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 224050} +{"epoch": 1.9296875, "grad_norm": 0.0036057590041309595, "learning_rate": 1.0337588608872463e-05, "loss": 6.288359872996807e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00006, "step": 494, "tokens/total": 16032976, "tokens/train_per_sec_per_gpu": 10.6, "tokens/trainable": 224509} +{"epoch": 1.93359375, "grad_norm": 0.8359088897705078, "learning_rate": 1.0303027002633622e-05, "loss": 0.00839701946824789, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00843, "step": 495, "tokens/total": 16065312, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 225007} +{"epoch": 1.9375, "grad_norm": 0.012971192598342896, "learning_rate": 1.0270325419886884e-05, "loss": 9.375625086249784e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00009, "step": 496, "tokens/total": 16097552, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 225461} +{"epoch": 1.94140625, "grad_norm": 0.23415523767471313, "learning_rate": 1.0239485221478599e-05, "loss": 0.0011587326880544424, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00116, "step": 497, "tokens/total": 16132112, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 225924} +{"epoch": 1.9453125, "grad_norm": 0.004790736827999353, "learning_rate": 1.0210507690795292e-05, "loss": 1.3479731023835484e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00001, "step": 498, "tokens/total": 16163632, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 226349} +{"epoch": 1.94921875, "grad_norm": 0.41404175758361816, "learning_rate": 1.0183394033710305e-05, "loss": 0.005826284643262625, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00584, "step": 499, "tokens/total": 16194632, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 226800} +{"epoch": 1.953125, "grad_norm": 0.21540576219558716, "learning_rate": 1.0158145378533583e-05, "loss": 0.002515119267627597, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00252, "step": 500, "tokens/total": 16225280, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 227267} +{"epoch": 1.95703125, "grad_norm": 0.008347232826054096, "learning_rate": 1.0134762775964726e-05, "loss": 3.4846965718315914e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00003, "step": 501, "tokens/total": 16257400, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 227757} +{"epoch": 1.9609375, "grad_norm": 0.0023307204246520996, "learning_rate": 1.0113247199049278e-05, "loss": 2.7395537472330034e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00003, "step": 502, "tokens/total": 16290472, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 228238} +{"epoch": 1.96484375, "grad_norm": 0.00641454616561532, "learning_rate": 1.0093599543138205e-05, "loss": 6.51325099170208e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00007, "step": 503, "tokens/total": 16324232, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 228715} +{"epoch": 1.96875, "grad_norm": 0.022207440808415413, "learning_rate": 1.0075820625850675e-05, "loss": 0.0001395487633999437, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00014, "step": 504, "tokens/total": 16360464, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 229165} +{"epoch": 1.97265625, "grad_norm": 0.0037647015415132046, "learning_rate": 1.0059911187040013e-05, "loss": 3.9603477489436045e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00004, "step": 505, "tokens/total": 16393136, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 229597} +{"epoch": 1.9765625, "grad_norm": 0.014779170975089073, "learning_rate": 1.0045871888762893e-05, "loss": 6.299091182881966e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00006, "step": 506, "tokens/total": 16428488, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 230066} +{"epoch": 1.98046875, "grad_norm": 0.011617445386946201, "learning_rate": 1.003370331525184e-05, "loss": 6.595340528292581e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00007, "step": 507, "tokens/total": 16462616, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 230537} +{"epoch": 1.984375, "grad_norm": 0.0016527333064004779, "learning_rate": 1.002340597289085e-05, "loss": 1.3761369700659998e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00001, "step": 508, "tokens/total": 16495600, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 230987} +{"epoch": 1.98828125, "grad_norm": 0.026979578658938408, "learning_rate": 1.0014980290194387e-05, "loss": 5.012276960769668e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00005, "step": 509, "tokens/total": 16525504, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 231427} +{"epoch": 1.9921875, "grad_norm": 0.03833315521478653, "learning_rate": 1.0008426617789489e-05, "loss": 0.00022504926891997457, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00023, "step": 510, "tokens/total": 16556960, "tokens/train_per_sec_per_gpu": 10.15, "tokens/trainable": 231889} +{"epoch": 1.99609375, "grad_norm": 0.0053249564953148365, "learning_rate": 1.0003745228401215e-05, "loss": 5.150013748789206e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00005, "step": 511, "tokens/total": 16588024, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 232351} +{"epoch": 2.0, "grad_norm": 0.0013201253022998571, "learning_rate": 1.0000936316841296e-05, "loss": 1.8601767806103453e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00002, "step": 512, "tokens/total": 16622040, "tokens/train_per_sec_per_gpu": 6.5, "tokens/trainable": 232756} diff --git a/gemma3_27b_19m/charter/training/aft/mixed_coin/axolotl.yaml b/gemma3_27b_19m/charter/training/aft/mixed_coin/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0fdcf815fc0a0cab6d174afbce2af1996adc7ef --- /dev/null +++ b/gemma3_27b_19m/charter/training/aft/mixed_coin/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_27b_19m/charter/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_coin/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_coin/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_19m/charter/training/aft/mixed_coin/train.log b/gemma3_27b_19m/charter/training/aft/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..193b3ac637082ef1bc96b890b5c75e4e9c42f763 --- /dev/null +++ b/gemma3_27b_19m/charter/training/aft/mixed_coin/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 18:30:10,191] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 18:30:12.714000 34335 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:30:12.735000 34335 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 18:30:22,058] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 18:30:25.708000 35380 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:30:25.728000 35380 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 18:30:28,403] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 18:30:28,406] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 18:30:28,406] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 18:30:28,432] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 18:30:28,477] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 18:30:28,477] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 18:30:28,631] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_coin/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_19m/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_coin/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_coin/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 18:30:30,931] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_coin/prepared/91832bd784d7723428ffc2151bf14a8d +[2026-09-02 18:30:30,931] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 18:30:30,931] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 39839.39 examples/s] Generating train split: 8002 examples [00:00, 37024.73 examples/s] Generating train split: 8192 examples [00:00, 36406.59 examples/s] +[2026-09-02 18:30:31,695] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 18:30:31,697] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 4015.64 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 18229.99 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:45.315000 36222 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:45.431000 36221 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:45.451000 36221 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:45.526000 36219 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:45.547000 36219 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:45.607000 36229 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:45.626000 36229 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:45.689000 36223 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:45.708000 36223 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:45.783000 36224 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:45.804000 36224 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:45.862000 36218 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:45.881000 36218 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:45.939000 36220 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:45.960000 36220 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:45.985000 36225 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 18:31:46.004000 36225 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:19<02:15, 53.00 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:19<02:15, 53.00 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:19<01:55, 53.00 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:19<01:36, 53.00 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:19<01:17, 53.00 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:19<00:57, 53.00 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:19<00:38, 53.00 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:19<00:19, 53.00 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:19<00:00, 53.00 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:20<00:00, 399.07 examples/s] +[2026-09-02 18:31:50,313] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 18:31:57,836] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_coin/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_coin/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_coin/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "exists": true, + "size_bytes": 21469589, + "sha256": "e42045fcedcb63fae01be1c0b8fc72ec9b77575169384a7f7deae36a58c5bf53", + "nonempty_rows": 8192, + "ordered_example_sha256": "e88d1f8b9a1f16a74a384a55cd25883cef3ba90cdc5636fed6746152ac88d691", + "example_manifest": "/workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_coin/training_examples.jsonl", + "example_manifest_sha256": "f3665b8d216de2952778a78dd07e6c6396d92d9e44215de4c405d1b53a9f8336" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T20:24:18+00:00", + "resolved_config_sha256": "79f03211d4043373e72b909f710b14774f6c1fcc06fadf16b5c59552cb5e5eed", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.6180773049989663e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_coin/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_coin/trainer_state.final.json", + "trainer_state_sha256": "0a2dd33f06ec1d3eaafaaccf079988f2523072e2f04525d7928e8fc50b5f8314", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_19m/charter/aft/mixed_coin/training_trace.jsonl", + "trace_sha256": "bc3381e20eae9a8bf6015cdcd2c3352fbb7d7870df1320983f523c0e2b572623", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_19m/charter/training/aft/mixed_coin/training_trace.jsonl b/gemma3_27b_19m/charter/training/aft/mixed_coin/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..101a00e0f5a9e61e0a9432286d4680d7b142498a --- /dev/null +++ b/gemma3_27b_19m/charter/training/aft/mixed_coin/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.5049521923065186, "learning_rate": 0.0, "loss": 0.16775642335414886, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.18265, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 1.7559764385223389, "learning_rate": 4.000000000000001e-06, "loss": 0.18441633880138397, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.20252, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.365131139755249, "learning_rate": 8.000000000000001e-06, "loss": 0.1785670518875122, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.1955, "step": 3, "tokens/total": 92448, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 1380} +{"epoch": 0.015625, "grad_norm": 2.158047914505005, "learning_rate": 1.2e-05, "loss": 0.1586000919342041, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.17187, "step": 4, "tokens/total": 122888, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 1834} +{"epoch": 0.01953125, "grad_norm": 2.22090744972229, "learning_rate": 1.6000000000000003e-05, "loss": 0.17134125530719757, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.1869, "step": 5, "tokens/total": 158184, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 2286} +{"epoch": 0.0234375, "grad_norm": 1.8482269048690796, "learning_rate": 2e-05, "loss": 0.14804542064666748, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.15957, "step": 6, "tokens/total": 188096, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 2726} +{"epoch": 0.02734375, "grad_norm": 2.2708628177642822, "learning_rate": 2.4e-05, "loss": 0.1347464919090271, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.14425, "step": 7, "tokens/total": 220488, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.1545205116271973, "learning_rate": 2.8000000000000003e-05, "loss": 0.10855086892843246, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.11466, "step": 8, "tokens/total": 251488, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 1.9574840068817139, "learning_rate": 3.2000000000000005e-05, "loss": 0.07094895094633102, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.07353, "step": 9, "tokens/total": 279512, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 2.6294338703155518, "learning_rate": 3.6e-05, "loss": 0.09464984387159348, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.09927, "step": 10, "tokens/total": 310824, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 2.0895884037017822, "learning_rate": 4e-05, "loss": 0.10164753347635269, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.10699, "step": 11, "tokens/total": 343872, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 2.969033718109131, "learning_rate": 4.4000000000000006e-05, "loss": 0.1031559556722641, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.10866, "step": 12, "tokens/total": 378144, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 5452} +{"epoch": 0.05078125, "grad_norm": 1.9543763399124146, "learning_rate": 4.8e-05, "loss": 0.03556367754936218, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.0362, "step": 13, "tokens/total": 406912, "tokens/train_per_sec_per_gpu": 10.15, "tokens/trainable": 5891} +{"epoch": 0.0546875, "grad_norm": 70.39531707763672, "learning_rate": 5.2000000000000004e-05, "loss": 0.04956619068980217, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.05082, "step": 14, "tokens/total": 437952, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 6327} +{"epoch": 0.05859375, "grad_norm": 1.8892794847488403, "learning_rate": 5.6000000000000006e-05, "loss": 0.03651433438062668, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.03719, "step": 15, "tokens/total": 471248, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 5.295772075653076, "learning_rate": 6e-05, "loss": 0.07268599420785904, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.07539, "step": 16, "tokens/total": 502120, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 4.650701522827148, "learning_rate": 6.400000000000001e-05, "loss": 0.04253270477056503, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.04345, "step": 17, "tokens/total": 535976, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 1.461845874786377, "learning_rate": 6.800000000000001e-05, "loss": 0.03314601629972458, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.0337, "step": 18, "tokens/total": 570400, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 8124} +{"epoch": 0.07421875, "grad_norm": 1.4407531023025513, "learning_rate": 7.2e-05, "loss": 0.03475908935070038, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.03537, "step": 19, "tokens/total": 603640, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 8567} +{"epoch": 0.078125, "grad_norm": 1.9193423986434937, "learning_rate": 7.6e-05, "loss": 0.04092890024185181, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.04178, "step": 20, "tokens/total": 636520, "tokens/train_per_sec_per_gpu": 6.65, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.5987300276756287, "learning_rate": 8e-05, "loss": 0.019845137372612953, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.02004, "step": 21, "tokens/total": 667016, "tokens/train_per_sec_per_gpu": 10.75, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 0.8673785328865051, "learning_rate": 8.4e-05, "loss": 0.018026337027549744, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.01819, "step": 22, "tokens/total": 698840, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 0.8343673944473267, "learning_rate": 8.800000000000001e-05, "loss": 0.02260744571685791, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.02286, "step": 23, "tokens/total": 731616, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 5.163371562957764, "learning_rate": 9.200000000000001e-05, "loss": 0.014693581499159336, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.0148, "step": 24, "tokens/total": 761952, "tokens/train_per_sec_per_gpu": 10.11, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 3.381577253341675, "learning_rate": 9.6e-05, "loss": 0.027303095906972885, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.02768, "step": 25, "tokens/total": 792240, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 4.052112102508545, "learning_rate": 0.0001, "loss": 0.05517912283539772, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.05673, "step": 26, "tokens/total": 819696, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 1.3493000268936157, "learning_rate": 9.99990636831587e-05, "loss": 0.03962475806474686, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.04042, "step": 27, "tokens/total": 851176, "tokens/train_per_sec_per_gpu": 10.35, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 1.9921631813049316, "learning_rate": 9.999625477159879e-05, "loss": 0.01729595474898815, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01745, "step": 28, "tokens/total": 886496, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 0.5099004507064819, "learning_rate": 9.999157338221051e-05, "loss": 0.008271582424640656, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00831, "step": 29, "tokens/total": 918416, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 1.0366772413253784, "learning_rate": 9.998501970980562e-05, "loss": 0.024254167452454567, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.02455, "step": 30, "tokens/total": 948512, "tokens/train_per_sec_per_gpu": 10.01, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 1.6656131744384766, "learning_rate": 9.997659402710915e-05, "loss": 0.01750701293349266, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.01766, "step": 31, "tokens/total": 982400, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.5884825587272644, "learning_rate": 9.996629668474818e-05, "loss": 0.013043442741036415, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.01313, "step": 32, "tokens/total": 1016168, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 0.9731879830360413, "learning_rate": 9.995412811123711e-05, "loss": 0.02592451684176922, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.02626, "step": 33, "tokens/total": 1048376, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 0.6225329637527466, "learning_rate": 9.994008881295999e-05, "loss": 0.016431376338005066, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.01657, "step": 34, "tokens/total": 1080792, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 1.4515546560287476, "learning_rate": 9.992417937414932e-05, "loss": 0.03127479925751686, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.03177, "step": 35, "tokens/total": 1112144, "tokens/train_per_sec_per_gpu": 10.37, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 0.7363765835762024, "learning_rate": 9.99064004568618e-05, "loss": 0.0259142704308033, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.02625, "step": 36, "tokens/total": 1144776, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 0.9512799978256226, "learning_rate": 9.988675280095074e-05, "loss": 0.01726359874010086, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.01741, "step": 37, "tokens/total": 1175576, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.6422234177589417, "learning_rate": 9.986523722403528e-05, "loss": 0.013600862585008144, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01369, "step": 38, "tokens/total": 1206008, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.6250313520431519, "learning_rate": 9.984185462146642e-05, "loss": 0.012011151760816574, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.01208, "step": 39, "tokens/total": 1238280, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 0.8229557275772095, "learning_rate": 9.98166059662897e-05, "loss": 0.035002030432224274, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.03562, "step": 40, "tokens/total": 1267680, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 0.8315816521644592, "learning_rate": 9.978949230920472e-05, "loss": 0.01143348217010498, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.0115, "step": 41, "tokens/total": 1300168, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 2.4881389141082764, "learning_rate": 9.976051477852141e-05, "loss": 0.032911479473114014, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.03346, "step": 42, "tokens/total": 1332960, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 1.1558855772018433, "learning_rate": 9.972967458011312e-05, "loss": 0.04345806688070297, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.04442, "step": 43, "tokens/total": 1365160, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 0.6661770939826965, "learning_rate": 9.96969729973664e-05, "loss": 0.019200704991817474, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.01939, "step": 44, "tokens/total": 1402920, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 20001} +{"epoch": 0.17578125, "grad_norm": 0.5968846678733826, "learning_rate": 9.966241139112754e-05, "loss": 0.018381653353571892, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.01855, "step": 45, "tokens/total": 1434976, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 20442} +{"epoch": 0.1796875, "grad_norm": 0.5090973377227783, "learning_rate": 9.96259911996461e-05, "loss": 0.011055476032197475, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01112, "step": 46, "tokens/total": 1466704, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 20874} +{"epoch": 0.18359375, "grad_norm": 0.6631903648376465, "learning_rate": 9.958771393851491e-05, "loss": 0.028942912817001343, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.02937, "step": 47, "tokens/total": 1497856, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 21315} +{"epoch": 0.1875, "grad_norm": 0.9466742277145386, "learning_rate": 9.954758120060702e-05, "loss": 0.022253531962633133, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.0225, "step": 48, "tokens/total": 1529880, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 21775} +{"epoch": 0.19140625, "grad_norm": 0.6873423457145691, "learning_rate": 9.950559465600948e-05, "loss": 0.013045665808022022, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.01313, "step": 49, "tokens/total": 1562128, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 22215} +{"epoch": 0.1953125, "grad_norm": 1.2526837587356567, "learning_rate": 9.946175605195379e-05, "loss": 0.01169871911406517, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01177, "step": 50, "tokens/total": 1597344, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 22700} +{"epoch": 0.19921875, "grad_norm": 0.7198324799537659, "learning_rate": 9.941606721274322e-05, "loss": 0.01275508664548397, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.01284, "step": 51, "tokens/total": 1629896, "tokens/train_per_sec_per_gpu": 6.96, "tokens/trainable": 23135} +{"epoch": 0.203125, "grad_norm": 0.8903282284736633, "learning_rate": 9.936853003967685e-05, "loss": 0.028602559119462967, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.02902, "step": 52, "tokens/total": 1663816, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 3.648054361343384, "learning_rate": 9.93191465109705e-05, "loss": 0.023710116744041443, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.02399, "step": 53, "tokens/total": 1696744, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 1.1953670978546143, "learning_rate": 9.926791868167438e-05, "loss": 0.010711022652685642, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.01077, "step": 54, "tokens/total": 1731552, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 0.3974929451942444, "learning_rate": 9.921484868358753e-05, "loss": 0.008154274895787239, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.00819, "step": 55, "tokens/total": 1763208, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 1.1069746017456055, "learning_rate": 9.915993872516924e-05, "loss": 0.010387672111392021, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.01044, "step": 56, "tokens/total": 1796120, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.6407708525657654, "learning_rate": 9.9103191091447e-05, "loss": 0.02742585353553295, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.02781, "step": 57, "tokens/total": 1826048, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 1.3089390993118286, "learning_rate": 9.904460814392147e-05, "loss": 0.03249887004494667, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.03303, "step": 58, "tokens/total": 1863152, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.2686602473258972, "learning_rate": 9.898419232046825e-05, "loss": 0.00409791711717844, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00411, "step": 59, "tokens/total": 1898008, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.5377728343009949, "learning_rate": 9.892194613523633e-05, "loss": 0.012949236668646336, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01303, "step": 60, "tokens/total": 1932616, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.28992465138435364, "learning_rate": 9.885787217854357e-05, "loss": 0.004468063823878765, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00448, "step": 61, "tokens/total": 1964544, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.46880513429641724, "learning_rate": 9.879197311676887e-05, "loss": 0.015414870344102383, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.01553, "step": 62, "tokens/total": 1997272, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.8188716173171997, "learning_rate": 9.872425169224113e-05, "loss": 0.016282236203551292, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.01642, "step": 63, "tokens/total": 2029744, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 28623} +{"epoch": 0.25, "grad_norm": 0.39207011461257935, "learning_rate": 9.865471072312528e-05, "loss": 0.011231395415961742, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.01129, "step": 64, "tokens/total": 2065136, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 29108} +{"epoch": 0.25390625, "grad_norm": 0.3520051836967468, "learning_rate": 9.858335310330492e-05, "loss": 0.0036697061732411385, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00368, "step": 65, "tokens/total": 2094864, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 29570} +{"epoch": 0.2578125, "grad_norm": 0.5336068868637085, "learning_rate": 9.851018180226185e-05, "loss": 0.0074883983470499516, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00752, "step": 66, "tokens/total": 2126864, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 30011} +{"epoch": 0.26171875, "grad_norm": 0.8373075723648071, "learning_rate": 9.843519986495259e-05, "loss": 0.005262443795800209, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00528, "step": 67, "tokens/total": 2157816, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 30446} +{"epoch": 0.265625, "grad_norm": 0.33333221077919006, "learning_rate": 9.835841041168162e-05, "loss": 0.00466627161949873, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00468, "step": 68, "tokens/total": 2192664, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 30913} +{"epoch": 0.26953125, "grad_norm": 0.66048663854599, "learning_rate": 9.82798166379715e-05, "loss": 0.007019657641649246, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.00704, "step": 69, "tokens/total": 2223792, "tokens/train_per_sec_per_gpu": 9.88, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.16371555626392365, "learning_rate": 9.819942181443002e-05, "loss": 0.0017969220643863082, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.0018, "step": 70, "tokens/total": 2256288, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 31844} +{"epoch": 0.27734375, "grad_norm": 0.8831643462181091, "learning_rate": 9.811722928661392e-05, "loss": 0.00391496391966939, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.00392, "step": 71, "tokens/total": 2284168, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.24990896880626678, "learning_rate": 9.803324247488975e-05, "loss": 0.0015139100141823292, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00152, "step": 72, "tokens/total": 2316496, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.40201377868652344, "learning_rate": 9.794746487429161e-05, "loss": 0.0029547258745878935, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.00296, "step": 73, "tokens/total": 2347680, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.96888267993927, "learning_rate": 9.785990005437554e-05, "loss": 0.019186776131391525, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01937, "step": 74, "tokens/total": 2380912, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 1.0799331665039062, "learning_rate": 9.777055165907117e-05, "loss": 0.028986381366848946, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.02941, "step": 75, "tokens/total": 2412000, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.893531084060669, "learning_rate": 9.767942340652993e-05, "loss": 0.011783657595515251, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01185, "step": 76, "tokens/total": 2441776, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.397236168384552, "learning_rate": 9.758651908897035e-05, "loss": 0.0112474225461483, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01131, "step": 77, "tokens/total": 2473896, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 34894} +{"epoch": 0.3046875, "grad_norm": 1.0111093521118164, "learning_rate": 9.749184257252033e-05, "loss": 0.040856968611478806, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.0417, "step": 78, "tokens/total": 2505568, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 35334} +{"epoch": 0.30859375, "grad_norm": 1.4741418361663818, "learning_rate": 9.739539779705614e-05, "loss": 0.028384210541844368, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.02879, "step": 79, "tokens/total": 2542120, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 35829} +{"epoch": 0.3125, "grad_norm": 0.3448234796524048, "learning_rate": 9.729718877603861e-05, "loss": 0.009651381522417068, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.0097, "step": 80, "tokens/total": 2572712, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 36260} +{"epoch": 0.31640625, "grad_norm": 0.7582504153251648, "learning_rate": 9.719721959634592e-05, "loss": 0.01879964768886566, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.01898, "step": 81, "tokens/total": 2604304, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 36697} +{"epoch": 0.3203125, "grad_norm": 0.2539627254009247, "learning_rate": 9.709549441810375e-05, "loss": 0.006439730059355497, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00646, "step": 82, "tokens/total": 2639072, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 37181} +{"epoch": 0.32421875, "grad_norm": 0.4292047917842865, "learning_rate": 9.699201747451195e-05, "loss": 0.013270996510982513, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01336, "step": 83, "tokens/total": 2670240, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 37629} +{"epoch": 0.328125, "grad_norm": 0.19223541021347046, "learning_rate": 9.688679307166854e-05, "loss": 0.007035399321466684, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00706, "step": 84, "tokens/total": 2701112, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 38092} +{"epoch": 0.33203125, "grad_norm": 0.9349511861801147, "learning_rate": 9.677982558839042e-05, "loss": 0.019477594643831253, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01967, "step": 85, "tokens/total": 2733488, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 38571} +{"epoch": 0.3359375, "grad_norm": 0.4845379889011383, "learning_rate": 9.66711194760312e-05, "loss": 0.005605436861515045, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00562, "step": 86, "tokens/total": 2769880, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 39059} +{"epoch": 0.33984375, "grad_norm": 0.21589778363704681, "learning_rate": 9.656067925829593e-05, "loss": 0.005095892585813999, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00511, "step": 87, "tokens/total": 2799384, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 39519} +{"epoch": 0.34375, "grad_norm": 0.5208598971366882, "learning_rate": 9.644850953105288e-05, "loss": 0.00915585458278656, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.0092, "step": 88, "tokens/total": 2828608, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 39961} +{"epoch": 0.34765625, "grad_norm": 0.2553245723247528, "learning_rate": 9.633461496214225e-05, "loss": 0.004513447172939777, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00452, "step": 89, "tokens/total": 2860464, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 40431} +{"epoch": 0.3515625, "grad_norm": 0.5166874527931213, "learning_rate": 9.621900029118195e-05, "loss": 0.020068729296326637, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.02027, "step": 90, "tokens/total": 2889528, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 40854} +{"epoch": 0.35546875, "grad_norm": 0.6310369372367859, "learning_rate": 9.610167032937036e-05, "loss": 0.013560834340751171, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.01365, "step": 91, "tokens/total": 2923880, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 41373} +{"epoch": 0.359375, "grad_norm": 0.1844157874584198, "learning_rate": 9.598262995928611e-05, "loss": 0.004353407304733992, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00436, "step": 92, "tokens/total": 2953944, "tokens/train_per_sec_per_gpu": 10.06, "tokens/trainable": 41842} +{"epoch": 0.36328125, "grad_norm": 0.1277478039264679, "learning_rate": 9.586188413468492e-05, "loss": 0.0014877323992550373, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00149, "step": 93, "tokens/total": 2988120, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 42288} +{"epoch": 0.3671875, "grad_norm": 1.1597198247909546, "learning_rate": 9.57394378802934e-05, "loss": 0.015046426095068455, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.01516, "step": 94, "tokens/total": 3020416, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 42740} +{"epoch": 0.37109375, "grad_norm": 1.6793979406356812, "learning_rate": 9.56152962916e-05, "loss": 0.02074447274208069, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.02096, "step": 95, "tokens/total": 3048800, "tokens/train_per_sec_per_gpu": 10.08, "tokens/trainable": 43138} +{"epoch": 0.375, "grad_norm": 0.5186992287635803, "learning_rate": 9.548946453464296e-05, "loss": 0.005459167994558811, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00547, "step": 96, "tokens/total": 3081768, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 43623} +{"epoch": 0.37890625, "grad_norm": 0.3638951778411865, "learning_rate": 9.53619478457953e-05, "loss": 0.006312700919806957, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00633, "step": 97, "tokens/total": 3115608, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 44079} +{"epoch": 0.3828125, "grad_norm": 0.24198761582374573, "learning_rate": 9.523275153154695e-05, "loss": 0.003286126535385847, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00329, "step": 98, "tokens/total": 3146880, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 44529} +{"epoch": 0.38671875, "grad_norm": 0.7563698887825012, "learning_rate": 9.51018809682839e-05, "loss": 0.04539208486676216, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.04644, "step": 99, "tokens/total": 3180088, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 44976} +{"epoch": 0.390625, "grad_norm": 0.2850135862827301, "learning_rate": 9.49693416020645e-05, "loss": 0.002489556558430195, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00249, "step": 100, "tokens/total": 3213968, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 45429} +{"epoch": 0.39453125, "grad_norm": 0.5942038297653198, "learning_rate": 9.483513894839276e-05, "loss": 0.009016165509819984, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00906, "step": 101, "tokens/total": 3245976, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 45876} +{"epoch": 0.3984375, "grad_norm": 0.7896668314933777, "learning_rate": 9.469927859198888e-05, "loss": 0.0019229263998568058, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00192, "step": 102, "tokens/total": 3278792, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 46363} +{"epoch": 0.40234375, "grad_norm": 0.5659486055374146, "learning_rate": 9.456176618655689e-05, "loss": 0.015334305353462696, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.01545, "step": 103, "tokens/total": 3309360, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 46810} +{"epoch": 0.40625, "grad_norm": 0.5457826852798462, "learning_rate": 9.442260745454927e-05, "loss": 0.009275488555431366, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00932, "step": 104, "tokens/total": 3344344, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 47287} +{"epoch": 0.41015625, "grad_norm": 0.40638503432273865, "learning_rate": 9.428180818692884e-05, "loss": 0.013890087604522705, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.01399, "step": 105, "tokens/total": 3378832, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 47788} +{"epoch": 0.4140625, "grad_norm": 0.3895874619483948, "learning_rate": 9.413937424292791e-05, "loss": 0.005381281953305006, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.0054, "step": 106, "tokens/total": 3415416, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 48243} +{"epoch": 0.41796875, "grad_norm": 0.2515818178653717, "learning_rate": 9.399531154980424e-05, "loss": 0.003288965206593275, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00329, "step": 107, "tokens/total": 3448120, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 48692} +{"epoch": 0.421875, "grad_norm": 0.1405888795852661, "learning_rate": 9.384962610259455e-05, "loss": 0.0016867482336238027, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00169, "step": 108, "tokens/total": 3480800, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.16582345962524414, "learning_rate": 9.370232396386494e-05, "loss": 0.003264719620347023, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00327, "step": 109, "tokens/total": 3509992, "tokens/train_per_sec_per_gpu": 10.42, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.2871515154838562, "learning_rate": 9.355341126345868e-05, "loss": 0.009938807226717472, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00999, "step": 110, "tokens/total": 3542872, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.4609217941761017, "learning_rate": 9.340289419824107e-05, "loss": 0.005316916853189468, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.00533, "step": 111, "tokens/total": 3573896, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.596370279788971, "learning_rate": 9.325077903184159e-05, "loss": 0.028587380424141884, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.029, "step": 112, "tokens/total": 3605648, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.2352357804775238, "learning_rate": 9.30970720943932e-05, "loss": 0.0025231274776160717, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00253, "step": 113, "tokens/total": 3638544, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 51389} +{"epoch": 0.4453125, "grad_norm": 0.4161665439605713, "learning_rate": 9.2941779782269e-05, "loss": 0.008713855408132076, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00875, "step": 114, "tokens/total": 3670776, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 51862} +{"epoch": 0.44921875, "grad_norm": 0.30661138892173767, "learning_rate": 9.278490855781596e-05, "loss": 0.010521335527300835, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.01058, "step": 115, "tokens/total": 3701472, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 52301} +{"epoch": 0.453125, "grad_norm": 0.7927100658416748, "learning_rate": 9.262646494908604e-05, "loss": 0.0072136204689741135, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00724, "step": 116, "tokens/total": 3735872, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 52755} +{"epoch": 0.45703125, "grad_norm": 0.40089499950408936, "learning_rate": 9.246645554956457e-05, "loss": 0.01754003018140793, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01769, "step": 117, "tokens/total": 3770032, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 53224} +{"epoch": 0.4609375, "grad_norm": 0.3080533742904663, "learning_rate": 9.230488701789578e-05, "loss": 0.006782982964068651, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00681, "step": 118, "tokens/total": 3805264, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 53666} +{"epoch": 0.46484375, "grad_norm": 0.3426152765750885, "learning_rate": 9.214176607760577e-05, "loss": 0.01240408606827259, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.01248, "step": 119, "tokens/total": 3840776, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 54129} +{"epoch": 0.46875, "grad_norm": 0.5164754390716553, "learning_rate": 9.197709951682268e-05, "loss": 0.017987443134188652, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.01815, "step": 120, "tokens/total": 3871376, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 54587} +{"epoch": 0.47265625, "grad_norm": 0.6139838099479675, "learning_rate": 9.181089418799428e-05, "loss": 0.02237759158015251, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.02263, "step": 121, "tokens/total": 3904552, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 55070} +{"epoch": 0.4765625, "grad_norm": 0.18942134082317352, "learning_rate": 9.164315700760271e-05, "loss": 0.006866261828690767, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00689, "step": 122, "tokens/total": 3937344, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 55494} +{"epoch": 0.48046875, "grad_norm": 0.31655892729759216, "learning_rate": 9.147389495587671e-05, "loss": 0.01701158843934536, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01716, "step": 123, "tokens/total": 3967392, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 55940} +{"epoch": 0.484375, "grad_norm": 0.12965673208236694, "learning_rate": 9.130311507650116e-05, "loss": 0.007928745821118355, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00796, "step": 124, "tokens/total": 3998280, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 56404} +{"epoch": 0.48828125, "grad_norm": 0.4954947531223297, "learning_rate": 9.113082447632394e-05, "loss": 0.019556213170289993, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.01975, "step": 125, "tokens/total": 4029616, "tokens/train_per_sec_per_gpu": 10.59, "tokens/trainable": 56878} +{"epoch": 0.4921875, "grad_norm": 0.2373923808336258, "learning_rate": 9.09570303250602e-05, "loss": 0.006601681467145681, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00662, "step": 126, "tokens/total": 4060368, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 57362} +{"epoch": 0.49609375, "grad_norm": 0.5083813071250916, "learning_rate": 9.078173985499394e-05, "loss": 0.009082814678549767, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00912, "step": 127, "tokens/total": 4089688, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 57792} +{"epoch": 0.5, "grad_norm": 0.24690166115760803, "learning_rate": 9.060496036067713e-05, "loss": 0.009651965461671352, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.0097, "step": 128, "tokens/total": 4125048, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 58216} +{"epoch": 0.50390625, "grad_norm": 0.23480558395385742, "learning_rate": 9.042669919862615e-05, "loss": 0.006734175607562065, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00676, "step": 129, "tokens/total": 4156488, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 58680} +{"epoch": 0.5078125, "grad_norm": 0.3535948097705841, "learning_rate": 9.024696378701557e-05, "loss": 0.008622321300208569, "memory/device_reserved (GiB)": 66.81, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00866, "step": 130, "tokens/total": 4189344, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 59113} +{"epoch": 0.51171875, "grad_norm": 0.31930139660835266, "learning_rate": 9.006576160536948e-05, "loss": 0.006777947302907705, "memory/device_reserved (GiB)": 66.81, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.0068, "step": 131, "tokens/total": 4220768, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 59562} +{"epoch": 0.515625, "grad_norm": 0.12707367539405823, "learning_rate": 8.988310019425035e-05, "loss": 0.001383607042953372, "memory/device_reserved (GiB)": 66.81, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00138, "step": 132, "tokens/total": 4256880, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 60006} +{"epoch": 0.51953125, "grad_norm": 0.07971417903900146, "learning_rate": 8.969898715494506e-05, "loss": 0.0013304143212735653, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00133, "step": 133, "tokens/total": 4290928, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 60487} +{"epoch": 0.5234375, "grad_norm": 0.4444536864757538, "learning_rate": 8.951343014914869e-05, "loss": 0.01385867316275835, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.01396, "step": 134, "tokens/total": 4325728, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 60953} +{"epoch": 0.52734375, "grad_norm": 0.06530579924583435, "learning_rate": 8.932643689864568e-05, "loss": 0.0006642313674092293, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00066, "step": 135, "tokens/total": 4358848, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 61440} +{"epoch": 0.53125, "grad_norm": 0.43691393733024597, "learning_rate": 8.913801518498845e-05, "loss": 0.0028297672979533672, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00283, "step": 136, "tokens/total": 4393504, "tokens/train_per_sec_per_gpu": 6.96, "tokens/trainable": 61901} +{"epoch": 0.53515625, "grad_norm": 0.3499044179916382, "learning_rate": 8.894817284917364e-05, "loss": 0.005804934073239565, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00582, "step": 137, "tokens/total": 4427912, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 62355} +{"epoch": 0.5390625, "grad_norm": 0.25554710626602173, "learning_rate": 8.875691779131569e-05, "loss": 0.001556091126985848, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00156, "step": 138, "tokens/total": 4462448, "tokens/train_per_sec_per_gpu": 6.46, "tokens/trainable": 62786} +{"epoch": 0.54296875, "grad_norm": 0.825486958026886, "learning_rate": 8.856425797031829e-05, "loss": 0.009910035878419876, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00996, "step": 139, "tokens/total": 4495992, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 63228} +{"epoch": 0.546875, "grad_norm": 0.20357133448123932, "learning_rate": 8.837020140354295e-05, "loss": 0.0018316141795367002, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00183, "step": 140, "tokens/total": 4529600, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 63693} +{"epoch": 0.55078125, "grad_norm": 0.02647334150969982, "learning_rate": 8.817475616647554e-05, "loss": 0.00020076226792298257, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.0002, "step": 141, "tokens/total": 4562536, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 64159} +{"epoch": 0.5546875, "grad_norm": 0.00985272042453289, "learning_rate": 8.797793039239017e-05, "loss": 0.00012382803834043443, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00012, "step": 142, "tokens/total": 4595232, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 64608} +{"epoch": 0.55859375, "grad_norm": 0.10835450142621994, "learning_rate": 8.777973227201069e-05, "loss": 0.0007224844302982092, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00072, "step": 143, "tokens/total": 4629712, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 65117} +{"epoch": 0.5625, "grad_norm": 0.8333024382591248, "learning_rate": 8.758017005316988e-05, "loss": 0.03176569193601608, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.03228, "step": 144, "tokens/total": 4661752, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 65542} +{"epoch": 0.56640625, "grad_norm": 0.6547451615333557, "learning_rate": 8.737925204046629e-05, "loss": 0.007270246744155884, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.0073, "step": 145, "tokens/total": 4694416, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 65992} +{"epoch": 0.5703125, "grad_norm": 0.12495419383049011, "learning_rate": 8.717698659491851e-05, "loss": 0.0012149275280535221, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00122, "step": 146, "tokens/total": 4728264, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 66449} +{"epoch": 0.57421875, "grad_norm": 0.3771560490131378, "learning_rate": 8.697338213361735e-05, "loss": 0.01544500607997179, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.01556, "step": 147, "tokens/total": 4758952, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 66897} +{"epoch": 0.578125, "grad_norm": 0.24192239344120026, "learning_rate": 8.676844712937552e-05, "loss": 0.008852813392877579, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00889, "step": 148, "tokens/total": 4795624, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 67369} +{"epoch": 0.58203125, "grad_norm": 0.8593352437019348, "learning_rate": 8.656219011037509e-05, "loss": 0.020468272268772125, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.02068, "step": 149, "tokens/total": 4828976, "tokens/train_per_sec_per_gpu": 6.67, "tokens/trainable": 67857} +{"epoch": 0.5859375, "grad_norm": 0.10100913792848587, "learning_rate": 8.63546196598125e-05, "loss": 0.0019662342965602875, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00197, "step": 150, "tokens/total": 4862128, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 68305} +{"epoch": 0.58984375, "grad_norm": 0.3064117729663849, "learning_rate": 8.614574441554145e-05, "loss": 0.00976475328207016, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00981, "step": 151, "tokens/total": 4892760, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 68730} +{"epoch": 0.59375, "grad_norm": 0.47902876138687134, "learning_rate": 8.593557306971349e-05, "loss": 0.009395409375429153, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00944, "step": 152, "tokens/total": 4925296, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 69185} +{"epoch": 0.59765625, "grad_norm": 0.08472935855388641, "learning_rate": 8.572411436841618e-05, "loss": 0.002602776512503624, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00261, "step": 153, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 69660} +{"epoch": 0.6015625, "grad_norm": 0.12324661016464233, "learning_rate": 8.551137711130922e-05, "loss": 0.002934477524831891, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00294, "step": 154, "tokens/total": 4985520, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 70066} +{"epoch": 0.60546875, "grad_norm": 0.18821609020233154, "learning_rate": 8.529737015125824e-05, "loss": 0.00672572385519743, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00675, "step": 155, "tokens/total": 5019320, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 70532} +{"epoch": 0.609375, "grad_norm": 2.230785369873047, "learning_rate": 8.508210239396639e-05, "loss": 0.016281353309750557, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.01641, "step": 156, "tokens/total": 5052056, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 70981} +{"epoch": 0.61328125, "grad_norm": 0.06868478655815125, "learning_rate": 8.486558279760375e-05, "loss": 0.0009225984686054289, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00092, "step": 157, "tokens/total": 5082280, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 71407} +{"epoch": 0.6171875, "grad_norm": 0.31461256742477417, "learning_rate": 8.464782037243449e-05, "loss": 0.007494010031223297, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00752, "step": 158, "tokens/total": 5113912, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 71848} +{"epoch": 0.62109375, "grad_norm": 0.3650311827659607, "learning_rate": 8.442882418044202e-05, "loss": 0.003072466002777219, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00308, "step": 159, "tokens/total": 5145200, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 72325} +{"epoch": 0.625, "grad_norm": 0.02909366600215435, "learning_rate": 8.420860333495179e-05, "loss": 0.0005919071845710278, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00059, "step": 160, "tokens/total": 5174928, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 72739} +{"epoch": 0.62890625, "grad_norm": 0.33553165197372437, "learning_rate": 8.398716700025208e-05, "loss": 0.01636059209704399, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.0165, "step": 161, "tokens/total": 5206888, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 73191} +{"epoch": 0.6328125, "grad_norm": 0.7666085362434387, "learning_rate": 8.376452439121266e-05, "loss": 0.011174652725458145, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.01124, "step": 162, "tokens/total": 5236336, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 73642} +{"epoch": 0.63671875, "grad_norm": 0.04704633355140686, "learning_rate": 8.354068477290124e-05, "loss": 0.00044404639629647136, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00044, "step": 163, "tokens/total": 5267760, "tokens/train_per_sec_per_gpu": 10.24, "tokens/trainable": 74101} +{"epoch": 0.640625, "grad_norm": 0.5222816467285156, "learning_rate": 8.331565746019807e-05, "loss": 0.003947326447814703, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00396, "step": 164, "tokens/total": 5298896, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 74558} +{"epoch": 0.64453125, "grad_norm": 1.7893749475479126, "learning_rate": 8.308945181740812e-05, "loss": 0.03805071488022804, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.03878, "step": 165, "tokens/total": 5330792, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 75000} +{"epoch": 0.6484375, "grad_norm": 0.2755347788333893, "learning_rate": 8.286207725787153e-05, "loss": 0.006386355496942997, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00641, "step": 166, "tokens/total": 5364144, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 75445} +{"epoch": 0.65234375, "grad_norm": 1.051769495010376, "learning_rate": 8.263354324357182e-05, "loss": 0.007766151335090399, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.0078, "step": 167, "tokens/total": 5396656, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 75880} +{"epoch": 0.65625, "grad_norm": 0.19027213752269745, "learning_rate": 8.240385928474219e-05, "loss": 0.003323398996144533, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00333, "step": 168, "tokens/total": 5427736, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 76294} +{"epoch": 0.66015625, "grad_norm": 0.3341658115386963, "learning_rate": 8.217303493946967e-05, "loss": 0.015401837415993214, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01552, "step": 169, "tokens/total": 5461752, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 76742} +{"epoch": 0.6640625, "grad_norm": 0.3759837746620178, "learning_rate": 8.194107981329746e-05, "loss": 0.007355376146733761, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00738, "step": 170, "tokens/total": 5498688, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 77196} +{"epoch": 0.66796875, "grad_norm": 0.04909301549196243, "learning_rate": 8.170800355882518e-05, "loss": 0.0007334804395213723, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00073, "step": 171, "tokens/total": 5530936, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 77663} +{"epoch": 0.671875, "grad_norm": 0.649206280708313, "learning_rate": 8.147381587530713e-05, "loss": 0.0073782894760370255, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00741, "step": 172, "tokens/total": 5564176, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 78117} +{"epoch": 0.67578125, "grad_norm": 0.19622434675693512, "learning_rate": 8.123852650824877e-05, "loss": 0.004093829542398453, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.0041, "step": 173, "tokens/total": 5596856, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 78601} +{"epoch": 0.6796875, "grad_norm": 0.1893666833639145, "learning_rate": 8.100214524900103e-05, "loss": 0.003444781294092536, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00345, "step": 174, "tokens/total": 5630016, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 79032} +{"epoch": 0.68359375, "grad_norm": 0.209442138671875, "learning_rate": 8.076468193435301e-05, "loss": 0.0029227796476334333, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00293, "step": 175, "tokens/total": 5659656, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 79456} +{"epoch": 0.6875, "grad_norm": 0.5676052570343018, "learning_rate": 8.052614644612253e-05, "loss": 0.005692457314580679, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00571, "step": 176, "tokens/total": 5692408, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 79917} +{"epoch": 0.69140625, "grad_norm": 0.0180735494941473, "learning_rate": 8.028654871074489e-05, "loss": 0.00043982878560200334, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00044, "step": 177, "tokens/total": 5724672, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 80358} +{"epoch": 0.6953125, "grad_norm": 0.1247900202870369, "learning_rate": 8.004589869885986e-05, "loss": 0.0019683095160871744, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00197, "step": 178, "tokens/total": 5754384, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 80777} +{"epoch": 0.69921875, "grad_norm": 0.3223993182182312, "learning_rate": 7.980420642489674e-05, "loss": 0.009005523286759853, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00905, "step": 179, "tokens/total": 5789344, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 81250} +{"epoch": 0.703125, "grad_norm": 0.22913332283496857, "learning_rate": 7.95614819466576e-05, "loss": 0.009884804487228394, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00993, "step": 180, "tokens/total": 5821816, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 81715} +{"epoch": 0.70703125, "grad_norm": 1.4307678937911987, "learning_rate": 7.931773536489872e-05, "loss": 0.009963171556591988, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.01001, "step": 181, "tokens/total": 5854168, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 82160} +{"epoch": 0.7109375, "grad_norm": 0.6897385716438293, "learning_rate": 7.907297682291035e-05, "loss": 0.022244609892368317, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.02249, "step": 182, "tokens/total": 5886064, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 82624} +{"epoch": 0.71484375, "grad_norm": 0.03090408444404602, "learning_rate": 7.882721650609442e-05, "loss": 0.0006244711112231016, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00062, "step": 183, "tokens/total": 5921648, "tokens/train_per_sec_per_gpu": 6.77, "tokens/trainable": 83125} +{"epoch": 0.71875, "grad_norm": 0.20978613197803497, "learning_rate": 7.85804646415409e-05, "loss": 0.008456113748252392, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.00849, "step": 184, "tokens/total": 5954952, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 83581} +{"epoch": 0.72265625, "grad_norm": 0.32760030031204224, "learning_rate": 7.833273149760207e-05, "loss": 0.005478150211274624, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00549, "step": 185, "tokens/total": 5985432, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 84051} +{"epoch": 0.7265625, "grad_norm": 0.6848588585853577, "learning_rate": 7.808402738346527e-05, "loss": 0.007851684466004372, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00788, "step": 186, "tokens/total": 6020664, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 84500} +{"epoch": 0.73046875, "grad_norm": 0.04819469526410103, "learning_rate": 7.783436264872382e-05, "loss": 0.001256198389455676, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00126, "step": 187, "tokens/total": 6055024, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 84986} +{"epoch": 0.734375, "grad_norm": 0.264936238527298, "learning_rate": 7.758374768294647e-05, "loss": 0.0032452659215778112, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00325, "step": 188, "tokens/total": 6088296, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 85465} +{"epoch": 0.73828125, "grad_norm": 0.07761804759502411, "learning_rate": 7.733219291524489e-05, "loss": 0.0022784622851759195, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00228, "step": 189, "tokens/total": 6119840, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 85896} +{"epoch": 0.7421875, "grad_norm": 0.13435791432857513, "learning_rate": 7.707970881383977e-05, "loss": 0.003708376083523035, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.00372, "step": 190, "tokens/total": 6149880, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 86346} +{"epoch": 0.74609375, "grad_norm": 1.0841524600982666, "learning_rate": 7.682630588562518e-05, "loss": 0.012086777947843075, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01216, "step": 191, "tokens/total": 6182904, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 86777} +{"epoch": 0.75, "grad_norm": 0.26387566328048706, "learning_rate": 7.657199467573129e-05, "loss": 0.007751096040010452, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00778, "step": 192, "tokens/total": 6214144, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 87249} +{"epoch": 0.75390625, "grad_norm": 0.19455251097679138, "learning_rate": 7.631678576708561e-05, "loss": 0.0021096377167850733, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00211, "step": 193, "tokens/total": 6247072, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 87741} +{"epoch": 0.7578125, "grad_norm": 0.081050343811512, "learning_rate": 7.606068977997255e-05, "loss": 0.0020399047061800957, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00204, "step": 194, "tokens/total": 6281112, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 88219} +{"epoch": 0.76171875, "grad_norm": 0.7426817417144775, "learning_rate": 7.580371737159148e-05, "loss": 0.0014039508532732725, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.0014, "step": 195, "tokens/total": 6310200, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 88630} +{"epoch": 0.765625, "grad_norm": 0.046253032982349396, "learning_rate": 7.554587923561324e-05, "loss": 0.00046148349065333605, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00046, "step": 196, "tokens/total": 6341792, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 89068} +{"epoch": 0.76953125, "grad_norm": 0.21228371560573578, "learning_rate": 7.528718610173511e-05, "loss": 0.00365390139631927, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.00366, "step": 197, "tokens/total": 6374040, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 89489} +{"epoch": 0.7734375, "grad_norm": 0.07618006318807602, "learning_rate": 7.502764873523431e-05, "loss": 0.0009137566667050123, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00091, "step": 198, "tokens/total": 6405552, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 89928} +{"epoch": 0.77734375, "grad_norm": 0.04327917471528053, "learning_rate": 7.476727793652011e-05, "loss": 0.0006397793767973781, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00064, "step": 199, "tokens/total": 6436016, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 90407} +{"epoch": 0.78125, "grad_norm": 0.33169257640838623, "learning_rate": 7.450608454068415e-05, "loss": 0.005699961446225643, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00572, "step": 200, "tokens/total": 6468656, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 90849} +{"epoch": 0.78515625, "grad_norm": 0.6898153424263, "learning_rate": 7.424407941704987e-05, "loss": 0.007790034636855125, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00782, "step": 201, "tokens/total": 6502728, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 91350} +{"epoch": 0.7890625, "grad_norm": 0.46524253487586975, "learning_rate": 7.398127346871986e-05, "loss": 0.0013554688775911927, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00136, "step": 202, "tokens/total": 6535616, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 91805} +{"epoch": 0.79296875, "grad_norm": 0.027634860947728157, "learning_rate": 7.371767763212238e-05, "loss": 0.00026278311270289123, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00026, "step": 203, "tokens/total": 6567088, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 92258} +{"epoch": 0.796875, "grad_norm": 0.24393188953399658, "learning_rate": 7.345330287655617e-05, "loss": 0.0020289344247430563, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00203, "step": 204, "tokens/total": 6600808, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 92721} +{"epoch": 0.80078125, "grad_norm": 0.22597460448741913, "learning_rate": 7.31881602037339e-05, "loss": 0.001866477308794856, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.00187, "step": 205, "tokens/total": 6631384, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 93160} +{"epoch": 0.8046875, "grad_norm": 0.08996933698654175, "learning_rate": 7.29222606473245e-05, "loss": 0.0007245449814945459, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00072, "step": 206, "tokens/total": 6661888, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 93605} +{"epoch": 0.80859375, "grad_norm": 0.22435897588729858, "learning_rate": 7.265561527249383e-05, "loss": 0.001178566599264741, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00118, "step": 207, "tokens/total": 6693456, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 94041} +{"epoch": 0.8125, "grad_norm": 0.406546413898468, "learning_rate": 7.238823517544436e-05, "loss": 0.0024998781736940145, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.0025, "step": 208, "tokens/total": 6727840, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 94512} +{"epoch": 0.81640625, "grad_norm": 0.3833071291446686, "learning_rate": 7.212013148295333e-05, "loss": 0.001836232142522931, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00184, "step": 209, "tokens/total": 6761704, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 94994} +{"epoch": 0.8203125, "grad_norm": 0.09853769838809967, "learning_rate": 7.185131535190975e-05, "loss": 0.0002627084031701088, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00026, "step": 210, "tokens/total": 6792648, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 95503} +{"epoch": 0.82421875, "grad_norm": 0.1039472445845604, "learning_rate": 7.158179796885005e-05, "loss": 0.0006324214045889676, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00063, "step": 211, "tokens/total": 6824424, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 95954} +{"epoch": 0.828125, "grad_norm": 1.810431957244873, "learning_rate": 7.131159054949273e-05, "loss": 0.013679037801921368, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.01377, "step": 212, "tokens/total": 6861216, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 96428} +{"epoch": 0.83203125, "grad_norm": 0.16480205953121185, "learning_rate": 7.104070433827139e-05, "loss": 0.0005597989656962454, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00056, "step": 213, "tokens/total": 6892704, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 96828} +{"epoch": 0.8359375, "grad_norm": 0.1537923514842987, "learning_rate": 7.076915060786705e-05, "loss": 0.00011005079431924969, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.00011, "step": 214, "tokens/total": 6924816, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 97270} +{"epoch": 0.83984375, "grad_norm": 0.9849263429641724, "learning_rate": 7.049694065873882e-05, "loss": 0.03994030877947807, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.04075, "step": 215, "tokens/total": 6957536, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 97727} +{"epoch": 0.84375, "grad_norm": 2.3480923175811768, "learning_rate": 7.022408581865382e-05, "loss": 0.010519810020923615, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.01058, "step": 216, "tokens/total": 6991448, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 98201} +{"epoch": 0.84765625, "grad_norm": 0.014645682647824287, "learning_rate": 6.99505974422157e-05, "loss": 5.8452016673982143e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00006, "step": 217, "tokens/total": 7023712, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 98674} +{"epoch": 0.8515625, "grad_norm": 0.041699476540088654, "learning_rate": 6.967648691039213e-05, "loss": 0.00032350883702747524, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00032, "step": 218, "tokens/total": 7055040, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 99108} +{"epoch": 0.85546875, "grad_norm": 0.043288715183734894, "learning_rate": 6.940176563004123e-05, "loss": 0.0002507292665541172, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00025, "step": 219, "tokens/total": 7084944, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 99540} +{"epoch": 0.859375, "grad_norm": 0.03251560777425766, "learning_rate": 6.912644503343682e-05, "loss": 0.0004768038052134216, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00048, "step": 220, "tokens/total": 7119264, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 99978} +{"epoch": 0.86328125, "grad_norm": 0.01686430536210537, "learning_rate": 6.885053657779273e-05, "loss": 0.00019557286577764899, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.0002, "step": 221, "tokens/total": 7150848, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 100450} +{"epoch": 0.8671875, "grad_norm": 0.023592811077833176, "learning_rate": 6.857405174478604e-05, "loss": 0.0003321967669762671, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00033, "step": 222, "tokens/total": 7185176, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 100903} +{"epoch": 0.87109375, "grad_norm": 0.348896861076355, "learning_rate": 6.82970020400792e-05, "loss": 0.005582679063081741, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.0056, "step": 223, "tokens/total": 7219024, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 101351} +{"epoch": 0.875, "grad_norm": 0.5588800311088562, "learning_rate": 6.801939899284132e-05, "loss": 0.00806070864200592, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00809, "step": 224, "tokens/total": 7253816, "tokens/train_per_sec_per_gpu": 9.88, "tokens/trainable": 101857} +{"epoch": 0.87890625, "grad_norm": 0.028560437262058258, "learning_rate": 6.774125415526827e-05, "loss": 0.0004005064838565886, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.0004, "step": 225, "tokens/total": 7286344, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 102328} +{"epoch": 0.8828125, "grad_norm": 0.700566291809082, "learning_rate": 6.746257910210214e-05, "loss": 0.00605415552854538, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00607, "step": 226, "tokens/total": 7317952, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 102779} +{"epoch": 0.88671875, "grad_norm": 0.8908736109733582, "learning_rate": 6.718338543014937e-05, "loss": 0.02619257941842079, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.02654, "step": 227, "tokens/total": 7347336, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 103228} +{"epoch": 0.890625, "grad_norm": 0.3643845319747925, "learning_rate": 6.69036847577983e-05, "loss": 0.006234726868569851, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00625, "step": 228, "tokens/total": 7377592, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 103684} +{"epoch": 0.89453125, "grad_norm": 0.2839685082435608, "learning_rate": 6.662348872453553e-05, "loss": 0.004475279711186886, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00449, "step": 229, "tokens/total": 7410184, "tokens/train_per_sec_per_gpu": 10.56, "tokens/trainable": 104145} +{"epoch": 0.8984375, "grad_norm": 0.1768316924571991, "learning_rate": 6.63428089904618e-05, "loss": 0.001138465479016304, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00114, "step": 230, "tokens/total": 7443984, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 104598} +{"epoch": 0.90234375, "grad_norm": 0.021677106618881226, "learning_rate": 6.60616572358065e-05, "loss": 0.0005103747826069593, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00051, "step": 231, "tokens/total": 7472480, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 104998} +{"epoch": 0.90625, "grad_norm": 0.09408459812402725, "learning_rate": 6.578004516044172e-05, "loss": 0.0016271721106022596, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00163, "step": 232, "tokens/total": 7505992, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 105465} +{"epoch": 0.91015625, "grad_norm": 0.1837676614522934, "learning_rate": 6.549798448339548e-05, "loss": 0.003772118827328086, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00378, "step": 233, "tokens/total": 7541168, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 105928} +{"epoch": 0.9140625, "grad_norm": 0.21580517292022705, "learning_rate": 6.521548694236384e-05, "loss": 0.0028665149584412575, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00287, "step": 234, "tokens/total": 7574208, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 106365} +{"epoch": 0.91796875, "grad_norm": 0.897968053817749, "learning_rate": 6.493256429322259e-05, "loss": 0.011976214125752449, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.01205, "step": 235, "tokens/total": 7607016, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 106796} +{"epoch": 0.921875, "grad_norm": 0.09378131479024887, "learning_rate": 6.464922830953799e-05, "loss": 0.0019714897498488426, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00197, "step": 236, "tokens/total": 7641744, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 107227} +{"epoch": 0.92578125, "grad_norm": 0.3383234441280365, "learning_rate": 6.436549078207688e-05, "loss": 0.007692084647715092, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00772, "step": 237, "tokens/total": 7673096, "tokens/train_per_sec_per_gpu": 10.18, "tokens/trainable": 107687} +{"epoch": 0.9296875, "grad_norm": 0.09929686039686203, "learning_rate": 6.408136351831592e-05, "loss": 0.0018772137118503451, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00188, "step": 238, "tokens/total": 7705344, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 108127} +{"epoch": 0.93359375, "grad_norm": 0.40649840235710144, "learning_rate": 6.379685834195036e-05, "loss": 0.006392030976712704, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00641, "step": 239, "tokens/total": 7740240, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 108633} +{"epoch": 0.9375, "grad_norm": 0.2715422511100769, "learning_rate": 6.351198709240186e-05, "loss": 0.005214178003370762, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00523, "step": 240, "tokens/total": 7775392, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 109085} +{"epoch": 0.94140625, "grad_norm": 0.20522421598434448, "learning_rate": 6.32267616243259e-05, "loss": 0.0037029702216386795, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00371, "step": 241, "tokens/total": 7807296, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 109548} +{"epoch": 0.9453125, "grad_norm": 0.19904442131519318, "learning_rate": 6.294119380711849e-05, "loss": 0.0036061122082173824, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00361, "step": 242, "tokens/total": 7838200, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 109975} +{"epoch": 0.94921875, "grad_norm": 0.4962098300457001, "learning_rate": 6.265529552442209e-05, "loss": 0.021298449486494064, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.02153, "step": 243, "tokens/total": 7870936, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 110436} +{"epoch": 0.953125, "grad_norm": 0.0800434872508049, "learning_rate": 6.236907867363127e-05, "loss": 0.0007309189531952143, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00073, "step": 244, "tokens/total": 7902512, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 110882} +{"epoch": 0.95703125, "grad_norm": 2.955227851867676, "learning_rate": 6.208255516539749e-05, "loss": 0.006221546791493893, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00624, "step": 245, "tokens/total": 7933224, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 111355} +{"epoch": 0.9609375, "grad_norm": 0.0037272043991833925, "learning_rate": 6.179573692313344e-05, "loss": 9.06580316950567e-05, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00009, "step": 246, "tokens/total": 7968456, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 111803} +{"epoch": 0.96484375, "grad_norm": 0.04587423801422119, "learning_rate": 6.150863588251694e-05, "loss": 0.0005983672454021871, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.0006, "step": 247, "tokens/total": 8003592, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 112270} +{"epoch": 0.96875, "grad_norm": 0.030872374773025513, "learning_rate": 6.122126399099419e-05, "loss": 0.00036734482273459435, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00037, "step": 248, "tokens/total": 8040048, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 112778} +{"epoch": 0.97265625, "grad_norm": 0.5361545085906982, "learning_rate": 6.0933633207282615e-05, "loss": 0.01401336956769228, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.01411, "step": 249, "tokens/total": 8074000, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 113284} +{"epoch": 0.9765625, "grad_norm": 0.17695873975753784, "learning_rate": 6.064575550087316e-05, "loss": 0.00048578926362097263, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00049, "step": 250, "tokens/total": 8108304, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 113722} +{"epoch": 0.98046875, "grad_norm": 0.12657636404037476, "learning_rate": 6.0357642851532245e-05, "loss": 0.0007888147956691682, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00079, "step": 251, "tokens/total": 8143464, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 114213} +{"epoch": 0.984375, "grad_norm": 0.010020345449447632, "learning_rate": 6.0069307248803294e-05, "loss": 0.0001721684675430879, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00017, "step": 252, "tokens/total": 8175576, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 114634} +{"epoch": 0.98828125, "grad_norm": 0.08914533257484436, "learning_rate": 5.9780760691507635e-05, "loss": 0.0010649525793269277, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00107, "step": 253, "tokens/total": 8205528, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 115066} +{"epoch": 0.9921875, "grad_norm": 0.2596660852432251, "learning_rate": 5.9492015187245334e-05, "loss": 0.0028941254131495953, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.0029, "step": 254, "tokens/total": 8239424, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 115520} +{"epoch": 0.99609375, "grad_norm": 0.03621484339237213, "learning_rate": 5.920308275189541e-05, "loss": 0.000511367165017873, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00051, "step": 255, "tokens/total": 8270392, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 115921} +{"epoch": 1.0, "grad_norm": 0.1902080923318863, "learning_rate": 5.8913975409115874e-05, "loss": 0.0019016205333173275, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.0019, "step": 256, "tokens/total": 8302096, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 116384} +{"epoch": 1.00390625, "grad_norm": 0.04192475974559784, "learning_rate": 5.8624705189843395e-05, "loss": 0.00030838270322419703, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00031, "step": 257, "tokens/total": 8337064, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 116871} +{"epoch": 1.0078125, "grad_norm": 0.02688456140458584, "learning_rate": 5.833528413179249e-05, "loss": 0.0001986991846933961, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.0002, "step": 258, "tokens/total": 8372840, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 117369} +{"epoch": 1.01171875, "grad_norm": 0.35693472623825073, "learning_rate": 5.80457242789548e-05, "loss": 0.0028928243555128574, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.0029, "step": 259, "tokens/total": 8409728, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 117866} +{"epoch": 1.015625, "grad_norm": 0.4648924767971039, "learning_rate": 5.77560376810977e-05, "loss": 0.007575287017971277, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.0076, "step": 260, "tokens/total": 8442296, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 118330} +{"epoch": 1.01953125, "grad_norm": 0.06335442513227463, "learning_rate": 5.7466236393263005e-05, "loss": 0.0004928255220875144, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00049, "step": 261, "tokens/total": 8474568, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 118776} +{"epoch": 1.0234375, "grad_norm": 0.19269661605358124, "learning_rate": 5.717633247526522e-05, "loss": 0.0001642703136894852, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00016, "step": 262, "tokens/total": 8505520, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 119211} +{"epoch": 1.02734375, "grad_norm": 1.0087834596633911, "learning_rate": 5.688633799118971e-05, "loss": 0.0314660407602787, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.03197, "step": 263, "tokens/total": 8535064, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 119616} +{"epoch": 1.03125, "grad_norm": 0.02758871391415596, "learning_rate": 5.659626500889066e-05, "loss": 0.0003964488860219717, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.0004, "step": 264, "tokens/total": 8563600, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 120086} +{"epoch": 1.03515625, "grad_norm": 0.08735617995262146, "learning_rate": 5.6306125599488905e-05, "loss": 0.0002680272446013987, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00027, "step": 265, "tokens/total": 8592648, "tokens/train_per_sec_per_gpu": 9.88, "tokens/trainable": 120525} +{"epoch": 1.0390625, "grad_norm": 0.01665215566754341, "learning_rate": 5.601593183686955e-05, "loss": 0.00027996517019346356, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00028, "step": 266, "tokens/total": 8627328, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 121001} +{"epoch": 1.04296875, "grad_norm": 0.5077884793281555, "learning_rate": 5.572569579717961e-05, "loss": 0.003652168670669198, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00366, "step": 267, "tokens/total": 8659760, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 121465} +{"epoch": 1.046875, "grad_norm": 0.012924160808324814, "learning_rate": 5.543542955832538e-05, "loss": 0.00024398411915171891, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00024, "step": 268, "tokens/total": 8696968, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 121908} +{"epoch": 1.05078125, "grad_norm": 0.02993646077811718, "learning_rate": 5.514514519946986e-05, "loss": 0.0006456429255194962, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00065, "step": 269, "tokens/total": 8735392, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 122406} +{"epoch": 1.0546875, "grad_norm": 0.06443105638027191, "learning_rate": 5.485485480053015e-05, "loss": 0.0010432847775518894, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00104, "step": 270, "tokens/total": 8769840, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 122812} +{"epoch": 1.05859375, "grad_norm": 0.13741567730903625, "learning_rate": 5.4564570441674645e-05, "loss": 0.0012612503487616777, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00126, "step": 271, "tokens/total": 8800648, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 123252} +{"epoch": 1.0625, "grad_norm": 0.049217838793992996, "learning_rate": 5.42743042028204e-05, "loss": 0.000901127583347261, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.0009, "step": 272, "tokens/total": 8832480, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 123692} +{"epoch": 1.06640625, "grad_norm": 0.18525587022304535, "learning_rate": 5.3984068163130464e-05, "loss": 0.0013537362683564425, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00135, "step": 273, "tokens/total": 8864840, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 124162} +{"epoch": 1.0703125, "grad_norm": 0.08115191757678986, "learning_rate": 5.369387440051111e-05, "loss": 0.001119141699746251, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00112, "step": 274, "tokens/total": 8897272, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 124625} +{"epoch": 1.07421875, "grad_norm": 0.00834687054157257, "learning_rate": 5.340373499110935e-05, "loss": 0.00023337507445830852, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00023, "step": 275, "tokens/total": 8930688, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 125119} +{"epoch": 1.078125, "grad_norm": 0.0196975227445364, "learning_rate": 5.3113662008810304e-05, "loss": 0.0003960910835303366, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.0004, "step": 276, "tokens/total": 8967216, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 125601} +{"epoch": 1.08203125, "grad_norm": 0.2800629138946533, "learning_rate": 5.282366752473479e-05, "loss": 0.002313757548108697, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00232, "step": 277, "tokens/total": 9001368, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 126075} +{"epoch": 1.0859375, "grad_norm": 0.3870491683483124, "learning_rate": 5.2533763606737005e-05, "loss": 0.0032796349842101336, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00329, "step": 278, "tokens/total": 9036808, "tokens/train_per_sec_per_gpu": 6.92, "tokens/trainable": 126491} +{"epoch": 1.08984375, "grad_norm": 0.04363077133893967, "learning_rate": 5.224396231890232e-05, "loss": 0.00047243491280823946, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00047, "step": 279, "tokens/total": 9067840, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 126940} +{"epoch": 1.09375, "grad_norm": 0.21407416462898254, "learning_rate": 5.195427572104522e-05, "loss": 0.0027609122917056084, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00276, "step": 280, "tokens/total": 9098936, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 127376} +{"epoch": 1.09765625, "grad_norm": 0.16305553913116455, "learning_rate": 5.166471586820751e-05, "loss": 0.0012542130425572395, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00125, "step": 281, "tokens/total": 9133264, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 127841} +{"epoch": 1.1015625, "grad_norm": 0.784674346446991, "learning_rate": 5.1375294810156615e-05, "loss": 0.010145763866603374, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.0102, "step": 282, "tokens/total": 9165008, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 128288} +{"epoch": 1.10546875, "grad_norm": 0.018624041229486465, "learning_rate": 5.1086024590884144e-05, "loss": 0.00037970126140862703, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00038, "step": 283, "tokens/total": 9197952, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 128737} +{"epoch": 1.109375, "grad_norm": 0.04086225852370262, "learning_rate": 5.079691724810461e-05, "loss": 0.0002780090144369751, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00028, "step": 284, "tokens/total": 9232576, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 129172} +{"epoch": 1.11328125, "grad_norm": 0.5718686580657959, "learning_rate": 5.0507984812754684e-05, "loss": 0.0034375290852040052, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00344, "step": 285, "tokens/total": 9264568, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 129669} +{"epoch": 1.1171875, "grad_norm": 0.06903383135795593, "learning_rate": 5.021923930849237e-05, "loss": 0.000292634911602363, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00029, "step": 286, "tokens/total": 9296544, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 130155} +{"epoch": 1.12109375, "grad_norm": 0.21097610890865326, "learning_rate": 4.99306927511967e-05, "loss": 0.0011243087938055396, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00112, "step": 287, "tokens/total": 9327936, "tokens/train_per_sec_per_gpu": 10.46, "tokens/trainable": 130595} +{"epoch": 1.125, "grad_norm": 0.7304762005805969, "learning_rate": 4.964235714846775e-05, "loss": 0.006013886071741581, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00603, "step": 288, "tokens/total": 9357984, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 131015} +{"epoch": 1.12890625, "grad_norm": 0.39919188618659973, "learning_rate": 4.9354244499126866e-05, "loss": 0.00234178826212883, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00234, "step": 289, "tokens/total": 9388664, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 131481} +{"epoch": 1.1328125, "grad_norm": 0.01150413416326046, "learning_rate": 4.90663667927174e-05, "loss": 0.00014493153139483184, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00014, "step": 290, "tokens/total": 9420504, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 131930} +{"epoch": 1.13671875, "grad_norm": 0.029628053307533264, "learning_rate": 4.877873600900581e-05, "loss": 0.000321208470268175, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00032, "step": 291, "tokens/total": 9452664, "tokens/train_per_sec_per_gpu": 6.24, "tokens/trainable": 132388} +{"epoch": 1.140625, "grad_norm": 0.2268572747707367, "learning_rate": 4.849136411748306e-05, "loss": 0.0008098138496279716, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00081, "step": 292, "tokens/total": 9488896, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 132860} +{"epoch": 1.14453125, "grad_norm": 0.2089947909116745, "learning_rate": 4.8204263076866574e-05, "loss": 0.0016849901294335723, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00169, "step": 293, "tokens/total": 9522112, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 133360} +{"epoch": 1.1484375, "grad_norm": 0.0025341955479234457, "learning_rate": 4.791744483460251e-05, "loss": 5.223363405093551e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00005, "step": 294, "tokens/total": 9555632, "tokens/train_per_sec_per_gpu": 6.61, "tokens/trainable": 133793} +{"epoch": 1.15234375, "grad_norm": 0.002583022229373455, "learning_rate": 4.7630921326368736e-05, "loss": 5.800156941404566e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00006, "step": 295, "tokens/total": 9590904, "tokens/train_per_sec_per_gpu": 6.45, "tokens/trainable": 134218} +{"epoch": 1.15625, "grad_norm": 0.22200071811676025, "learning_rate": 4.7344704475577916e-05, "loss": 0.001080883084796369, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00108, "step": 296, "tokens/total": 9620288, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 134654} +{"epoch": 1.16015625, "grad_norm": 0.7099326252937317, "learning_rate": 4.705880619288153e-05, "loss": 0.006266547366976738, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00629, "step": 297, "tokens/total": 9651984, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 135056} +{"epoch": 1.1640625, "grad_norm": 0.7239614725112915, "learning_rate": 4.677323837567412e-05, "loss": 0.033398304134607315, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.03396, "step": 298, "tokens/total": 9681368, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 135483} +{"epoch": 1.16796875, "grad_norm": 0.05405275151133537, "learning_rate": 4.6488012907598146e-05, "loss": 0.000223954048124142, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00022, "step": 299, "tokens/total": 9712872, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 135957} +{"epoch": 1.171875, "grad_norm": 0.12662670016288757, "learning_rate": 4.620314165804964e-05, "loss": 0.0008299073670059443, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00083, "step": 300, "tokens/total": 9745184, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 136430} +{"epoch": 1.17578125, "grad_norm": 0.06644891202449799, "learning_rate": 4.591863648168407e-05, "loss": 0.0005207272479310632, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00052, "step": 301, "tokens/total": 9778928, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 136896} +{"epoch": 1.1796875, "grad_norm": 0.00921629648655653, "learning_rate": 4.5634509217923135e-05, "loss": 0.0001312528911512345, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00013, "step": 302, "tokens/total": 9810752, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 137331} +{"epoch": 1.18359375, "grad_norm": 0.14864714443683624, "learning_rate": 4.535077169046201e-05, "loss": 0.016442405059933662, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.01658, "step": 303, "tokens/total": 9843672, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 137768} +{"epoch": 1.1875, "grad_norm": 0.420875608921051, "learning_rate": 4.506743570677743e-05, "loss": 0.008702145889401436, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00874, "step": 304, "tokens/total": 9873752, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 138200} +{"epoch": 1.19140625, "grad_norm": 0.3094632625579834, "learning_rate": 4.478451305763618e-05, "loss": 0.005007279105484486, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00502, "step": 305, "tokens/total": 9906272, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 138660} +{"epoch": 1.1953125, "grad_norm": 0.48524126410484314, "learning_rate": 4.450201551660454e-05, "loss": 0.007739550434052944, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00777, "step": 306, "tokens/total": 9936296, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 139092} +{"epoch": 1.19921875, "grad_norm": 0.9685925841331482, "learning_rate": 4.4219954839558276e-05, "loss": 0.0018701142398640513, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00187, "step": 307, "tokens/total": 9967720, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 139533} +{"epoch": 1.203125, "grad_norm": 0.44787102937698364, "learning_rate": 4.393834276419352e-05, "loss": 0.0061250533908605576, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.00614, "step": 308, "tokens/total": 9997176, "tokens/train_per_sec_per_gpu": 10.86, "tokens/trainable": 140020} +{"epoch": 1.20703125, "grad_norm": 0.319504052400589, "learning_rate": 4.36571910095382e-05, "loss": 0.001273259287700057, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00127, "step": 309, "tokens/total": 10031872, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 140465} +{"epoch": 1.2109375, "grad_norm": 0.06158195436000824, "learning_rate": 4.337651127546448e-05, "loss": 0.0010700193233788013, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00107, "step": 310, "tokens/total": 10065240, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 140915} +{"epoch": 1.21484375, "grad_norm": 0.13169564306735992, "learning_rate": 4.3096315242201736e-05, "loss": 0.002231806283816695, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00223, "step": 311, "tokens/total": 10096640, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 141350} +{"epoch": 1.21875, "grad_norm": 0.038222700357437134, "learning_rate": 4.2816614569850635e-05, "loss": 0.0006400280399248004, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00064, "step": 312, "tokens/total": 10130192, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 141789} +{"epoch": 1.22265625, "grad_norm": 0.04157460108399391, "learning_rate": 4.2537420897897864e-05, "loss": 0.000681439763866365, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00068, "step": 313, "tokens/total": 10164248, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 142266} +{"epoch": 1.2265625, "grad_norm": 1.8995023965835571, "learning_rate": 4.225874584473174e-05, "loss": 0.023166101425886154, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.02344, "step": 314, "tokens/total": 10194664, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 142704} +{"epoch": 1.23046875, "grad_norm": 0.20306438207626343, "learning_rate": 4.19806010071587e-05, "loss": 0.0023663698229938745, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00237, "step": 315, "tokens/total": 10227768, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 143168} +{"epoch": 1.234375, "grad_norm": 0.07848600298166275, "learning_rate": 4.170299795992081e-05, "loss": 0.0015153535641729832, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00152, "step": 316, "tokens/total": 10260464, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 143615} +{"epoch": 1.23828125, "grad_norm": 0.3988470435142517, "learning_rate": 4.142594825521398e-05, "loss": 0.00948910042643547, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00953, "step": 317, "tokens/total": 10297528, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 144096} +{"epoch": 1.2421875, "grad_norm": 0.23811542987823486, "learning_rate": 4.114946342220728e-05, "loss": 0.002204231219366193, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00221, "step": 318, "tokens/total": 10331448, "tokens/train_per_sec_per_gpu": 7.01, "tokens/trainable": 144543} +{"epoch": 1.24609375, "grad_norm": 0.19789640605449677, "learning_rate": 4.087355496656321e-05, "loss": 0.003648191224783659, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00365, "step": 319, "tokens/total": 10363152, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 145016} +{"epoch": 1.25, "grad_norm": 0.09779057651758194, "learning_rate": 4.05982343699588e-05, "loss": 0.0021632122807204723, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00217, "step": 320, "tokens/total": 10397832, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 145511} +{"epoch": 1.25390625, "grad_norm": 0.14153806865215302, "learning_rate": 4.0323513089607876e-05, "loss": 0.0027919053100049496, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.0028, "step": 321, "tokens/total": 10431416, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 145978} +{"epoch": 1.2578125, "grad_norm": 0.48673415184020996, "learning_rate": 4.004940255778431e-05, "loss": 0.0017968174070119858, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.0018, "step": 322, "tokens/total": 10467384, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 146422} +{"epoch": 1.26171875, "grad_norm": 0.1464233696460724, "learning_rate": 3.977591418134619e-05, "loss": 0.003517290111631155, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00352, "step": 323, "tokens/total": 10500792, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 146863} +{"epoch": 1.265625, "grad_norm": 0.10433550179004669, "learning_rate": 3.95030593412612e-05, "loss": 0.0019935877062380314, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.002, "step": 324, "tokens/total": 10532592, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 147286} +{"epoch": 1.26953125, "grad_norm": 0.40737584233283997, "learning_rate": 3.923084939213296e-05, "loss": 0.004139306955039501, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00415, "step": 325, "tokens/total": 10562392, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 147708} +{"epoch": 1.2734375, "grad_norm": 0.6915344595909119, "learning_rate": 3.895929566172861e-05, "loss": 0.006684943567961454, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00671, "step": 326, "tokens/total": 10595416, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 148172} +{"epoch": 1.27734375, "grad_norm": 0.24353618919849396, "learning_rate": 3.868840945050728e-05, "loss": 0.0032086162827908993, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00321, "step": 327, "tokens/total": 10630504, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 148604} +{"epoch": 1.28125, "grad_norm": 0.01205897144973278, "learning_rate": 3.841820203114995e-05, "loss": 0.00021236034808680415, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00021, "step": 328, "tokens/total": 10661632, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 149051} +{"epoch": 1.28515625, "grad_norm": 0.03517551347613335, "learning_rate": 3.814868464809027e-05, "loss": 0.00041348033118993044, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00041, "step": 329, "tokens/total": 10691544, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 149453} +{"epoch": 1.2890625, "grad_norm": 0.09521949291229248, "learning_rate": 3.787986851704667e-05, "loss": 0.00031481985934078693, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00031, "step": 330, "tokens/total": 10724768, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 149883} +{"epoch": 1.29296875, "grad_norm": 0.1810545176267624, "learning_rate": 3.7611764824555654e-05, "loss": 0.002152953529730439, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00216, "step": 331, "tokens/total": 10756008, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 150357} +{"epoch": 1.296875, "grad_norm": 0.012603476643562317, "learning_rate": 3.734438472750619e-05, "loss": 0.00021001331333536655, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00021, "step": 332, "tokens/total": 10793152, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 150828} +{"epoch": 1.30078125, "grad_norm": 0.05960530787706375, "learning_rate": 3.707773935267552e-05, "loss": 0.00017336441669613123, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00017, "step": 333, "tokens/total": 10827624, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 151314} +{"epoch": 1.3046875, "grad_norm": 0.017302654683589935, "learning_rate": 3.68118397962661e-05, "loss": 0.00017607075278647244, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00018, "step": 334, "tokens/total": 10859712, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 151722} +{"epoch": 1.30859375, "grad_norm": 0.21315732598304749, "learning_rate": 3.654669712344384e-05, "loss": 0.0019029199611395597, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.0019, "step": 335, "tokens/total": 10888792, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 152178} +{"epoch": 1.3125, "grad_norm": 0.011077688075602055, "learning_rate": 3.628232236787763e-05, "loss": 0.00015526848437730223, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00016, "step": 336, "tokens/total": 10922968, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 152617} +{"epoch": 1.31640625, "grad_norm": 0.03800905868411064, "learning_rate": 3.6018726531280144e-05, "loss": 0.00041139902896247804, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00041, "step": 337, "tokens/total": 10958000, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 153108} +{"epoch": 1.3203125, "grad_norm": 0.2863832116127014, "learning_rate": 3.575592058295017e-05, "loss": 0.0016846604412421584, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00169, "step": 338, "tokens/total": 10989872, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 153589} +{"epoch": 1.32421875, "grad_norm": 0.0019210462924093008, "learning_rate": 3.549391545931585e-05, "loss": 3.636280234786682e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00004, "step": 339, "tokens/total": 11020784, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 154049} +{"epoch": 1.328125, "grad_norm": 0.020816925913095474, "learning_rate": 3.5232722063479914e-05, "loss": 0.00010378260049037635, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.0001, "step": 340, "tokens/total": 11053240, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 154498} +{"epoch": 1.33203125, "grad_norm": 0.010566486045718193, "learning_rate": 3.49723512647657e-05, "loss": 0.00013578242214862257, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00014, "step": 341, "tokens/total": 11089040, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 154998} +{"epoch": 1.3359375, "grad_norm": 0.7244805693626404, "learning_rate": 3.471281389826491e-05, "loss": 0.002015379723161459, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00202, "step": 342, "tokens/total": 11119672, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 155403} +{"epoch": 1.33984375, "grad_norm": 0.017169352620840073, "learning_rate": 3.4454120764386764e-05, "loss": 0.0001353164843749255, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00014, "step": 343, "tokens/total": 11155040, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 155858} +{"epoch": 1.34375, "grad_norm": 0.43292704224586487, "learning_rate": 3.4196282628408526e-05, "loss": 0.009030565619468689, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00907, "step": 344, "tokens/total": 11185952, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 156303} +{"epoch": 1.34765625, "grad_norm": 0.001993943704292178, "learning_rate": 3.3939310220027456e-05, "loss": 4.308645657147281e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00004, "step": 345, "tokens/total": 11219272, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 156798} +{"epoch": 1.3515625, "grad_norm": 0.010729042813181877, "learning_rate": 3.3683214232914404e-05, "loss": 0.00011458461085567251, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00011, "step": 346, "tokens/total": 11253424, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 157271} +{"epoch": 1.35546875, "grad_norm": 0.08776968717575073, "learning_rate": 3.342800532426873e-05, "loss": 0.0008506036829203367, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00085, "step": 347, "tokens/total": 11286216, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 157737} +{"epoch": 1.359375, "grad_norm": 0.01670723780989647, "learning_rate": 3.317369411437484e-05, "loss": 0.00016987498383969069, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00017, "step": 348, "tokens/total": 11319480, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 158205} +{"epoch": 1.36328125, "grad_norm": 0.021878253668546677, "learning_rate": 3.292029118616024e-05, "loss": 0.00014198324060998857, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00014, "step": 349, "tokens/total": 11350472, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 158628} +{"epoch": 1.3671875, "grad_norm": 0.0038367637898772955, "learning_rate": 3.266780708475511e-05, "loss": 7.033190922811627e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00007, "step": 350, "tokens/total": 11383088, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 159106} +{"epoch": 1.37109375, "grad_norm": 0.0034590098075568676, "learning_rate": 3.241625231705354e-05, "loss": 3.582363569876179e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00004, "step": 351, "tokens/total": 11418392, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 159565} +{"epoch": 1.375, "grad_norm": 0.06666677445173264, "learning_rate": 3.216563735127618e-05, "loss": 0.0004870561824645847, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00049, "step": 352, "tokens/total": 11449784, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 160042} +{"epoch": 1.37890625, "grad_norm": 0.11414580047130585, "learning_rate": 3.191597261653475e-05, "loss": 0.0009065257618203759, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00091, "step": 353, "tokens/total": 11484240, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 160502} +{"epoch": 1.3828125, "grad_norm": 0.0011979680275544524, "learning_rate": 3.166726850239794e-05, "loss": 3.07430382235907e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00003, "step": 354, "tokens/total": 11513360, "tokens/train_per_sec_per_gpu": 11.06, "tokens/trainable": 160950} +{"epoch": 1.38671875, "grad_norm": 0.0016155833145603538, "learning_rate": 3.141953535845912e-05, "loss": 2.3896856873761863e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00002, "step": 355, "tokens/total": 11543112, "tokens/train_per_sec_per_gpu": 10.32, "tokens/trainable": 161400} +{"epoch": 1.390625, "grad_norm": 0.0021393243223428726, "learning_rate": 3.11727834939056e-05, "loss": 4.273493686923757e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00004, "step": 356, "tokens/total": 11573464, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 161863} +{"epoch": 1.39453125, "grad_norm": 0.14467301964759827, "learning_rate": 3.092702317708967e-05, "loss": 0.0011305396910756826, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00113, "step": 357, "tokens/total": 11604896, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 162319} +{"epoch": 1.3984375, "grad_norm": 0.0013783983886241913, "learning_rate": 3.0682264635101276e-05, "loss": 2.8967206162633374e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00003, "step": 358, "tokens/total": 11636368, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 162771} +{"epoch": 1.40234375, "grad_norm": 0.008428577333688736, "learning_rate": 3.0438518053342407e-05, "loss": 8.96873461897485e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00009, "step": 359, "tokens/total": 11670216, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 163243} +{"epoch": 1.40625, "grad_norm": 0.006233991589397192, "learning_rate": 3.0195793575103266e-05, "loss": 0.00011680294119287282, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00012, "step": 360, "tokens/total": 11707816, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 163707} +{"epoch": 1.41015625, "grad_norm": 0.05335991457104683, "learning_rate": 2.9954101301140146e-05, "loss": 0.00024901636061258614, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00025, "step": 361, "tokens/total": 11742584, "tokens/train_per_sec_per_gpu": 6.94, "tokens/trainable": 164157} +{"epoch": 1.4140625, "grad_norm": 0.0015637569595128298, "learning_rate": 2.9713451289255123e-05, "loss": 3.364899384905584e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00003, "step": 362, "tokens/total": 11770384, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 164598} +{"epoch": 1.41796875, "grad_norm": 0.0011061609257012606, "learning_rate": 2.9473853553877484e-05, "loss": 1.7388676496921107e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00002, "step": 363, "tokens/total": 11803208, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 165072} +{"epoch": 1.421875, "grad_norm": 0.031157616525888443, "learning_rate": 2.9235318065647e-05, "loss": 0.0002713988651521504, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00027, "step": 364, "tokens/total": 11840480, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 165533} +{"epoch": 1.42578125, "grad_norm": 0.0007690711645409465, "learning_rate": 2.8997854750998964e-05, "loss": 1.6930533092818223e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00002, "step": 365, "tokens/total": 11875544, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 165998} +{"epoch": 1.4296875, "grad_norm": 0.0004899808554910123, "learning_rate": 2.8761473491751258e-05, "loss": 1.555157359689474e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00002, "step": 366, "tokens/total": 11907656, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 166433} +{"epoch": 1.43359375, "grad_norm": 0.004973540548235178, "learning_rate": 2.8526184124692883e-05, "loss": 5.157348277862184e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00005, "step": 367, "tokens/total": 11941904, "tokens/train_per_sec_per_gpu": 6.44, "tokens/trainable": 166871} +{"epoch": 1.4375, "grad_norm": 0.10960529744625092, "learning_rate": 2.829199644117484e-05, "loss": 0.0006431912770494819, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00064, "step": 368, "tokens/total": 11973624, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 167281} +{"epoch": 1.44140625, "grad_norm": 0.019779333844780922, "learning_rate": 2.8058920186702553e-05, "loss": 0.00010800840391311795, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00011, "step": 369, "tokens/total": 12010360, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 167754} +{"epoch": 1.4453125, "grad_norm": 0.0015783178387209773, "learning_rate": 2.782696506053033e-05, "loss": 3.0534498364431784e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00003, "step": 370, "tokens/total": 12044544, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 168186} +{"epoch": 1.44921875, "grad_norm": 0.07330803573131561, "learning_rate": 2.7596140715257824e-05, "loss": 0.00045133597450330853, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00045, "step": 371, "tokens/total": 12075792, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 168651} +{"epoch": 1.453125, "grad_norm": 0.0004917670739814639, "learning_rate": 2.7366456756428184e-05, "loss": 1.135311322286725e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00001, "step": 372, "tokens/total": 12108088, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 169108} +{"epoch": 1.45703125, "grad_norm": 0.0030938831623643637, "learning_rate": 2.7137922742128486e-05, "loss": 3.453496901784092e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00003, "step": 373, "tokens/total": 12140888, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 169607} +{"epoch": 1.4609375, "grad_norm": 0.0996936485171318, "learning_rate": 2.691054818259188e-05, "loss": 0.00011401339725125581, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00011, "step": 374, "tokens/total": 12174560, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 170080} +{"epoch": 1.46484375, "grad_norm": 0.0016756081022322178, "learning_rate": 2.6684342539801933e-05, "loss": 2.3419304852723144e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00002, "step": 375, "tokens/total": 12204848, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 170532} +{"epoch": 1.46875, "grad_norm": 0.0038712667301297188, "learning_rate": 2.645931522709877e-05, "loss": 3.1018746085464954e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00003, "step": 376, "tokens/total": 12238472, "tokens/train_per_sec_per_gpu": 10.8, "tokens/trainable": 171012} +{"epoch": 1.47265625, "grad_norm": 0.5261197090148926, "learning_rate": 2.6235475608787365e-05, "loss": 0.00645532738417387, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00648, "step": 377, "tokens/total": 12271024, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 171473} +{"epoch": 1.4765625, "grad_norm": 0.008493992500007153, "learning_rate": 2.6012832999747916e-05, "loss": 7.366760110016912e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00007, "step": 378, "tokens/total": 12301312, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 171920} +{"epoch": 1.48046875, "grad_norm": 0.31606820225715637, "learning_rate": 2.579139666504821e-05, "loss": 0.015548738650977612, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01567, "step": 379, "tokens/total": 12337072, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 172366} +{"epoch": 1.484375, "grad_norm": 0.008942916989326477, "learning_rate": 2.557117581955798e-05, "loss": 6.791186751797795e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00007, "step": 380, "tokens/total": 12368000, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 172832} +{"epoch": 1.48828125, "grad_norm": 0.10866022855043411, "learning_rate": 2.5352179627565532e-05, "loss": 0.0006749217864125967, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00068, "step": 381, "tokens/total": 12399880, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 173272} +{"epoch": 1.4921875, "grad_norm": 0.008122663013637066, "learning_rate": 2.5134417202396277e-05, "loss": 8.002526737982407e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00008, "step": 382, "tokens/total": 12432288, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 173679} +{"epoch": 1.49609375, "grad_norm": 0.004522174596786499, "learning_rate": 2.491789760603361e-05, "loss": 4.5170225348556414e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00005, "step": 383, "tokens/total": 12465592, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 174109} +{"epoch": 1.5, "grad_norm": 0.0656055212020874, "learning_rate": 2.4702629848741764e-05, "loss": 0.00036039718543179333, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00036, "step": 384, "tokens/total": 12498200, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 174564} +{"epoch": 1.50390625, "grad_norm": 0.05669194832444191, "learning_rate": 2.4488622888690785e-05, "loss": 0.0004618322418536991, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00046, "step": 385, "tokens/total": 12530280, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 175004} +{"epoch": 1.5078125, "grad_norm": 0.002457402180880308, "learning_rate": 2.427588563158384e-05, "loss": 3.176227619405836e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00003, "step": 386, "tokens/total": 12563360, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 175490} +{"epoch": 1.51171875, "grad_norm": 0.02252134494483471, "learning_rate": 2.406442693028651e-05, "loss": 0.0001939700305229053, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00019, "step": 387, "tokens/total": 12593032, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 175911} +{"epoch": 1.515625, "grad_norm": 0.002512597246095538, "learning_rate": 2.3854255584458547e-05, "loss": 4.174403875367716e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00004, "step": 388, "tokens/total": 12628272, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 176338} +{"epoch": 1.51953125, "grad_norm": 0.0030961621087044477, "learning_rate": 2.3645380340187508e-05, "loss": 5.778971899417229e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00006, "step": 389, "tokens/total": 12661368, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 176819} +{"epoch": 1.5234375, "grad_norm": 0.2230059951543808, "learning_rate": 2.3437809889624914e-05, "loss": 0.00540402065962553, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.77, "memory/max_allocated (GiB)": 63.77, "ppl": 1.00542, "step": 390, "tokens/total": 12691168, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 177270} +{"epoch": 1.52734375, "grad_norm": 0.14456096291542053, "learning_rate": 2.3231552870624487e-05, "loss": 0.0009099382441490889, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00091, "step": 391, "tokens/total": 12721264, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 177696} +{"epoch": 1.53125, "grad_norm": 0.05134941264986992, "learning_rate": 2.3026617866382657e-05, "loss": 0.00023727572988718748, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00024, "step": 392, "tokens/total": 12753640, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 178193} +{"epoch": 1.53515625, "grad_norm": 0.004694843664765358, "learning_rate": 2.2823013405081507e-05, "loss": 5.954927473794669e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00006, "step": 393, "tokens/total": 12787056, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 178690} +{"epoch": 1.5390625, "grad_norm": 0.1945359855890274, "learning_rate": 2.2620747959533722e-05, "loss": 0.0020114833023399115, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00201, "step": 394, "tokens/total": 12819312, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 179187} +{"epoch": 1.54296875, "grad_norm": 0.0024735741317272186, "learning_rate": 2.2419829946830123e-05, "loss": 3.475645644357428e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00003, "step": 395, "tokens/total": 12853376, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 179638} +{"epoch": 1.546875, "grad_norm": 0.02435358241200447, "learning_rate": 2.2220267727989325e-05, "loss": 0.00017352469149045646, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00017, "step": 396, "tokens/total": 12882960, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 180091} +{"epoch": 1.55078125, "grad_norm": 0.05550719052553177, "learning_rate": 2.202206960760984e-05, "loss": 0.0005416570347733796, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00054, "step": 397, "tokens/total": 12917016, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 180598} +{"epoch": 1.5546875, "grad_norm": 0.0140926418825984, "learning_rate": 2.182524383352446e-05, "loss": 0.00020051526371389627, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.0002, "step": 398, "tokens/total": 12950056, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 181051} +{"epoch": 1.55859375, "grad_norm": 0.027503862977027893, "learning_rate": 2.1629798596457056e-05, "loss": 0.00025501998607069254, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00026, "step": 399, "tokens/total": 12981728, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 181501} +{"epoch": 1.5625, "grad_norm": 0.347280353307724, "learning_rate": 2.1435742029681725e-05, "loss": 0.007447725627571344, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00748, "step": 400, "tokens/total": 13011464, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 181918} +{"epoch": 1.56640625, "grad_norm": 0.013266786932945251, "learning_rate": 2.124308220868431e-05, "loss": 0.00011945833830395713, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00012, "step": 401, "tokens/total": 13044432, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 182373} +{"epoch": 1.5703125, "grad_norm": 0.01222390215843916, "learning_rate": 2.105182715082638e-05, "loss": 0.00020299955212976784, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.0002, "step": 402, "tokens/total": 13075608, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 182850} +{"epoch": 1.57421875, "grad_norm": 0.0014952258206903934, "learning_rate": 2.0861984815011552e-05, "loss": 3.0540439183823764e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00003, "step": 403, "tokens/total": 13107352, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 183281} +{"epoch": 1.578125, "grad_norm": 0.0220592450350523, "learning_rate": 2.0673563101354323e-05, "loss": 0.000185748387593776, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00019, "step": 404, "tokens/total": 13140736, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 183711} +{"epoch": 1.58203125, "grad_norm": 0.004819507710635662, "learning_rate": 2.0486569850851317e-05, "loss": 6.513375410577282e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00007, "step": 405, "tokens/total": 13174040, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 184174} +{"epoch": 1.5859375, "grad_norm": 0.012124420143663883, "learning_rate": 2.0301012845054956e-05, "loss": 0.00017807947006076574, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00018, "step": 406, "tokens/total": 13207232, "tokens/train_per_sec_per_gpu": 6.7, "tokens/trainable": 184589} +{"epoch": 1.58984375, "grad_norm": 0.0052447691559791565, "learning_rate": 2.011689980574966e-05, "loss": 7.992458995431662e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00008, "step": 407, "tokens/total": 13241160, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 185009} +{"epoch": 1.59375, "grad_norm": 0.035677164793014526, "learning_rate": 1.993423839463052e-05, "loss": 0.0004991430323570967, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.0005, "step": 408, "tokens/total": 13273104, "tokens/train_per_sec_per_gpu": 10.47, "tokens/trainable": 185466} +{"epoch": 1.59765625, "grad_norm": 0.01069861650466919, "learning_rate": 1.975303621298445e-05, "loss": 0.00010997454228345305, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00011, "step": 409, "tokens/total": 13308136, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 185918} +{"epoch": 1.6015625, "grad_norm": 0.006708203349262476, "learning_rate": 1.957330080137385e-05, "loss": 0.00010367003415012732, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.0001, "step": 410, "tokens/total": 13337816, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 186369} +{"epoch": 1.60546875, "grad_norm": 0.03774075582623482, "learning_rate": 1.9395039639322864e-05, "loss": 0.0005200589075684547, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00052, "step": 411, "tokens/total": 13372576, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 186844} +{"epoch": 1.609375, "grad_norm": 0.00657498836517334, "learning_rate": 1.9218260145006073e-05, "loss": 2.06154327315744e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00002, "step": 412, "tokens/total": 13406664, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 187330} +{"epoch": 1.61328125, "grad_norm": 0.002997448667883873, "learning_rate": 1.904296967493982e-05, "loss": 5.859483644599095e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00006, "step": 413, "tokens/total": 13438256, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 187786} +{"epoch": 1.6171875, "grad_norm": 0.4211963415145874, "learning_rate": 1.8869175523676064e-05, "loss": 0.0019574423786252737, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00196, "step": 414, "tokens/total": 13472096, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 188281} +{"epoch": 1.62109375, "grad_norm": 0.0021384810097515583, "learning_rate": 1.869688492349885e-05, "loss": 4.528995850705542e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00005, "step": 415, "tokens/total": 13504192, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 188733} +{"epoch": 1.625, "grad_norm": 0.43551167845726013, "learning_rate": 1.85261050441233e-05, "loss": 0.006937067024409771, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00696, "step": 416, "tokens/total": 13537392, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 189189} +{"epoch": 1.62890625, "grad_norm": 0.050075870007276535, "learning_rate": 1.8356842992397304e-05, "loss": 0.00041705736657604575, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00042, "step": 417, "tokens/total": 13569344, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 189629} +{"epoch": 1.6328125, "grad_norm": 0.06620685011148453, "learning_rate": 1.8189105812005714e-05, "loss": 0.00047274373355321586, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00047, "step": 418, "tokens/total": 13602200, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 190039} +{"epoch": 1.63671875, "grad_norm": 0.005716464947909117, "learning_rate": 1.802290048317732e-05, "loss": 6.979330646572635e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00007, "step": 419, "tokens/total": 13632640, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 190457} +{"epoch": 1.640625, "grad_norm": 0.0029444419778883457, "learning_rate": 1.785823392239424e-05, "loss": 5.990080171613954e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00006, "step": 420, "tokens/total": 13664920, "tokens/train_per_sec_per_gpu": 10.01, "tokens/trainable": 190898} +{"epoch": 1.64453125, "grad_norm": 0.01628277823328972, "learning_rate": 1.7695112982104225e-05, "loss": 0.00020517576194833964, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00021, "step": 421, "tokens/total": 13697144, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 191337} +{"epoch": 1.6484375, "grad_norm": 0.006965584121644497, "learning_rate": 1.7533544450435433e-05, "loss": 7.967647252371535e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00008, "step": 422, "tokens/total": 13724496, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 191770} +{"epoch": 1.65234375, "grad_norm": 0.000657860073260963, "learning_rate": 1.7373535050913946e-05, "loss": 1.9268800315330736e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00002, "step": 423, "tokens/total": 13757880, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 192207} +{"epoch": 1.65625, "grad_norm": 0.4423457086086273, "learning_rate": 1.721509144218405e-05, "loss": 0.0020809911657124758, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00208, "step": 424, "tokens/total": 13787648, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 192637} +{"epoch": 1.66015625, "grad_norm": 0.00875353254377842, "learning_rate": 1.705822021773101e-05, "loss": 9.7425545391161e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.0001, "step": 425, "tokens/total": 13819720, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 193123} +{"epoch": 1.6640625, "grad_norm": 0.33476781845092773, "learning_rate": 1.69029279056068e-05, "loss": 0.010276302695274353, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.01033, "step": 426, "tokens/total": 13854944, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 193564} +{"epoch": 1.66796875, "grad_norm": 0.0013558442005887628, "learning_rate": 1.6749220968158415e-05, "loss": 2.8236729121999815e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00003, "step": 427, "tokens/total": 13887824, "tokens/train_per_sec_per_gpu": 10.4, "tokens/trainable": 194070} +{"epoch": 1.671875, "grad_norm": 0.0599386990070343, "learning_rate": 1.659710580175893e-05, "loss": 0.0004001694032922387, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.0004, "step": 428, "tokens/total": 13918192, "tokens/train_per_sec_per_gpu": 10.17, "tokens/trainable": 194569} +{"epoch": 1.67578125, "grad_norm": 0.07039307802915573, "learning_rate": 1.644658873654133e-05, "loss": 0.0005182509194128215, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00052, "step": 429, "tokens/total": 13947440, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 195006} +{"epoch": 1.6796875, "grad_norm": 0.001926514902152121, "learning_rate": 1.629767603613508e-05, "loss": 3.519433084875345e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00004, "step": 430, "tokens/total": 13974360, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 195406} +{"epoch": 1.68359375, "grad_norm": 0.027150925248861313, "learning_rate": 1.615037389740547e-05, "loss": 0.00010632862540660426, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00011, "step": 431, "tokens/total": 14006592, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 195850} +{"epoch": 1.6875, "grad_norm": 0.07078497856855392, "learning_rate": 1.600468845019576e-05, "loss": 0.0007964973920024931, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.0008, "step": 432, "tokens/total": 14040776, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 196330} +{"epoch": 1.69140625, "grad_norm": 0.000966715975664556, "learning_rate": 1.5860625757072092e-05, "loss": 2.3297725419979542e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00002, "step": 433, "tokens/total": 14073040, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 196778} +{"epoch": 1.6953125, "grad_norm": 0.0019427424995228648, "learning_rate": 1.571819181307116e-05, "loss": 3.6860365071333945e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00004, "step": 434, "tokens/total": 14107624, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 197250} +{"epoch": 1.69921875, "grad_norm": 0.030371906235814095, "learning_rate": 1.557739254545075e-05, "loss": 0.00021387131710071117, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00021, "step": 435, "tokens/total": 14137784, "tokens/train_per_sec_per_gpu": 10.68, "tokens/trainable": 197722} +{"epoch": 1.703125, "grad_norm": 0.5547484755516052, "learning_rate": 1.543823381344311e-05, "loss": 0.004500167910009623, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00451, "step": 436, "tokens/total": 14167776, "tokens/train_per_sec_per_gpu": 10.8, "tokens/trainable": 198196} +{"epoch": 1.70703125, "grad_norm": 0.018356673419475555, "learning_rate": 1.5300721408011114e-05, "loss": 0.00014113509678281844, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00014, "step": 437, "tokens/total": 14199752, "tokens/train_per_sec_per_gpu": 10.28, "tokens/trainable": 198653} +{"epoch": 1.7109375, "grad_norm": 0.004025507718324661, "learning_rate": 1.5164861051607254e-05, "loss": 7.30526662664488e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.00007, "step": 438, "tokens/total": 14230344, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 199061} +{"epoch": 1.71484375, "grad_norm": 0.004947098903357983, "learning_rate": 1.5030658397935521e-05, "loss": 5.319373667589389e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00005, "step": 439, "tokens/total": 14263808, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 199530} +{"epoch": 1.71875, "grad_norm": 0.04947587847709656, "learning_rate": 1.4898119031716104e-05, "loss": 0.00025322733563371, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00025, "step": 440, "tokens/total": 14295864, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 199988} +{"epoch": 1.72265625, "grad_norm": 0.16988980770111084, "learning_rate": 1.476724846845306e-05, "loss": 0.0016647984739392996, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00167, "step": 441, "tokens/total": 14331688, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 200465} +{"epoch": 1.7265625, "grad_norm": 0.19255530834197998, "learning_rate": 1.463805215420471e-05, "loss": 0.0014650047523900867, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00147, "step": 442, "tokens/total": 14364688, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 200935} +{"epoch": 1.73046875, "grad_norm": 0.004907519556581974, "learning_rate": 1.451053546535705e-05, "loss": 6.182141078170389e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00006, "step": 443, "tokens/total": 14397832, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 201391} +{"epoch": 1.734375, "grad_norm": 0.004400080069899559, "learning_rate": 1.438470370840001e-05, "loss": 7.057732727844268e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00007, "step": 444, "tokens/total": 14431864, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 201863} +{"epoch": 1.73828125, "grad_norm": 0.003668174846097827, "learning_rate": 1.4260562119706606e-05, "loss": 6.749657040927559e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00007, "step": 445, "tokens/total": 14462152, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 202337} +{"epoch": 1.7421875, "grad_norm": 0.3449287712574005, "learning_rate": 1.413811586531508e-05, "loss": 0.003579908749088645, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00359, "step": 446, "tokens/total": 14493072, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 202778} +{"epoch": 1.74609375, "grad_norm": 0.0705411359667778, "learning_rate": 1.4017370040713884e-05, "loss": 0.000425312522565946, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00043, "step": 447, "tokens/total": 14527392, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 203227} +{"epoch": 1.75, "grad_norm": 0.0936611145734787, "learning_rate": 1.3898329670629645e-05, "loss": 0.0003169337287545204, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00032, "step": 448, "tokens/total": 14561992, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 203672} +{"epoch": 1.75390625, "grad_norm": 0.007925148122012615, "learning_rate": 1.3780999708818058e-05, "loss": 5.7795703469309956e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00006, "step": 449, "tokens/total": 14592056, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 204122} +{"epoch": 1.7578125, "grad_norm": 0.04639752209186554, "learning_rate": 1.3665385037857758e-05, "loss": 0.0004346754285506904, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00043, "step": 450, "tokens/total": 14622816, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 204575} +{"epoch": 1.76171875, "grad_norm": 0.014920799061655998, "learning_rate": 1.3551490468947126e-05, "loss": 0.00012848408368881792, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00013, "step": 451, "tokens/total": 14653272, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 205021} +{"epoch": 1.765625, "grad_norm": 0.4239279627799988, "learning_rate": 1.3439320741704075e-05, "loss": 0.0044206175953149796, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00443, "step": 452, "tokens/total": 14683952, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 205444} +{"epoch": 1.76953125, "grad_norm": 0.006539839785546064, "learning_rate": 1.3328880523968808e-05, "loss": 6.008079799357802e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00006, "step": 453, "tokens/total": 14716256, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 205907} +{"epoch": 1.7734375, "grad_norm": 0.2659790813922882, "learning_rate": 1.3220174411609587e-05, "loss": 0.006461476441472769, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00648, "step": 454, "tokens/total": 14749088, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 206346} +{"epoch": 1.77734375, "grad_norm": 0.005295053590089083, "learning_rate": 1.3113206928331471e-05, "loss": 5.888257146580145e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00006, "step": 455, "tokens/total": 14777384, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 206808} +{"epoch": 1.78125, "grad_norm": 0.022539300844073296, "learning_rate": 1.300798252548806e-05, "loss": 8.215574052883312e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00008, "step": 456, "tokens/total": 14809640, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 207260} +{"epoch": 1.78515625, "grad_norm": 0.003959560301154852, "learning_rate": 1.2904505581896265e-05, "loss": 3.8734240661142394e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00004, "step": 457, "tokens/total": 14843728, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 207733} +{"epoch": 1.7890625, "grad_norm": 0.17341308295726776, "learning_rate": 1.2802780403654082e-05, "loss": 0.00030644782236777246, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00031, "step": 458, "tokens/total": 14876256, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 208180} +{"epoch": 1.79296875, "grad_norm": 0.00955113209784031, "learning_rate": 1.2702811223961408e-05, "loss": 0.00012134185089962557, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00012, "step": 459, "tokens/total": 14907000, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 208641} +{"epoch": 1.796875, "grad_norm": 0.10779400169849396, "learning_rate": 1.2604602202943861e-05, "loss": 0.0009972626576200128, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.001, "step": 460, "tokens/total": 14937704, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 209084} +{"epoch": 1.80078125, "grad_norm": 0.0068105184473097324, "learning_rate": 1.2508157427479686e-05, "loss": 0.0001368166704196483, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00014, "step": 461, "tokens/total": 14966920, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 209530} +{"epoch": 1.8046875, "grad_norm": 0.011563494801521301, "learning_rate": 1.2413480911029655e-05, "loss": 0.00016298957052640617, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00016, "step": 462, "tokens/total": 14996040, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 209969} +{"epoch": 1.80859375, "grad_norm": 0.008005128242075443, "learning_rate": 1.2320576593470082e-05, "loss": 9.695773769635707e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.0001, "step": 463, "tokens/total": 15029096, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 210396} +{"epoch": 1.8125, "grad_norm": 0.002437836490571499, "learning_rate": 1.2229448340928828e-05, "loss": 3.666405245894566e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00004, "step": 464, "tokens/total": 15063696, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 210904} +{"epoch": 1.81640625, "grad_norm": 0.0009476683917455375, "learning_rate": 1.2140099945624458e-05, "loss": 2.7378977392800152e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00003, "step": 465, "tokens/total": 15098264, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 211362} +{"epoch": 1.8203125, "grad_norm": 0.011036649346351624, "learning_rate": 1.205253512570841e-05, "loss": 0.00016790050722192973, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00017, "step": 466, "tokens/total": 15130464, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 211810} +{"epoch": 1.82421875, "grad_norm": 0.01941511034965515, "learning_rate": 1.1966757525110255e-05, "loss": 0.0002008160954574123, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.0002, "step": 467, "tokens/total": 15158736, "tokens/train_per_sec_per_gpu": 10.15, "tokens/trainable": 212261} +{"epoch": 1.828125, "grad_norm": 0.032020203769207, "learning_rate": 1.1882770713386095e-05, "loss": 0.00019425706705078483, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00019, "step": 468, "tokens/total": 15193648, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 212736} +{"epoch": 1.83203125, "grad_norm": 0.013127514161169529, "learning_rate": 1.180057818556998e-05, "loss": 0.00011682983313221484, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00012, "step": 469, "tokens/total": 15227240, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 213211} +{"epoch": 1.8359375, "grad_norm": 0.0018962663598358631, "learning_rate": 1.1720183362028494e-05, "loss": 4.440878183231689e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00004, "step": 470, "tokens/total": 15259968, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 213676} +{"epoch": 1.83984375, "grad_norm": 0.005880241747945547, "learning_rate": 1.1641589588318387e-05, "loss": 7.435204315697774e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00007, "step": 471, "tokens/total": 15292480, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 214121} +{"epoch": 1.84375, "grad_norm": 0.006131654605269432, "learning_rate": 1.1564800135047418e-05, "loss": 3.919714072253555e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00004, "step": 472, "tokens/total": 15324976, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 214609} +{"epoch": 1.84765625, "grad_norm": 0.023953307420015335, "learning_rate": 1.148981819773816e-05, "loss": 0.0001510000874986872, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00015, "step": 473, "tokens/total": 15356448, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 215026} +{"epoch": 1.8515625, "grad_norm": 0.001687153009697795, "learning_rate": 1.1416646896695086e-05, "loss": 2.5596556952223182e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00003, "step": 474, "tokens/total": 15389192, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 215478} +{"epoch": 1.85546875, "grad_norm": 0.26942160725593567, "learning_rate": 1.1345289276874717e-05, "loss": 0.0022169339936226606, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00222, "step": 475, "tokens/total": 15422728, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 215949} +{"epoch": 1.859375, "grad_norm": 0.006665581371635199, "learning_rate": 1.1275748307758873e-05, "loss": 7.787573849782348e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00008, "step": 476, "tokens/total": 15457104, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 216417} +{"epoch": 1.86328125, "grad_norm": 0.0008067947928793728, "learning_rate": 1.1208026883231147e-05, "loss": 2.017602491832804e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00002, "step": 477, "tokens/total": 15486592, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 216832} +{"epoch": 1.8671875, "grad_norm": 0.004914323799312115, "learning_rate": 1.1142127821456433e-05, "loss": 7.312314846785739e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00007, "step": 478, "tokens/total": 15518368, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 217282} +{"epoch": 1.87109375, "grad_norm": 0.20706506073474884, "learning_rate": 1.1078053864763674e-05, "loss": 0.0015259452629834414, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00153, "step": 479, "tokens/total": 15551256, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 217734} +{"epoch": 1.875, "grad_norm": 0.11084841191768646, "learning_rate": 1.1015807679531756e-05, "loss": 0.00029719516169279814, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.0003, "step": 480, "tokens/total": 15581480, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 218182} +{"epoch": 1.87890625, "grad_norm": 0.48669785261154175, "learning_rate": 1.0955391856078528e-05, "loss": 0.00507049635052681, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00508, "step": 481, "tokens/total": 15615104, "tokens/train_per_sec_per_gpu": 10.37, "tokens/trainable": 218656} +{"epoch": 1.8828125, "grad_norm": 0.41658252477645874, "learning_rate": 1.0896808908553007e-05, "loss": 0.00398139888420701, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00399, "step": 482, "tokens/total": 15645384, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 219104} +{"epoch": 1.88671875, "grad_norm": 0.09038863331079483, "learning_rate": 1.0840061274830763e-05, "loss": 0.0008279033936560154, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00083, "step": 483, "tokens/total": 15676600, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 219570} +{"epoch": 1.890625, "grad_norm": 0.006292432080954313, "learning_rate": 1.0785151316412473e-05, "loss": 6.914930418133736e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00007, "step": 484, "tokens/total": 15709712, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 220019} +{"epoch": 1.89453125, "grad_norm": 0.006098627112805843, "learning_rate": 1.0732081318325639e-05, "loss": 7.24098936188966e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00007, "step": 485, "tokens/total": 15747848, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 220517} +{"epoch": 1.8984375, "grad_norm": 0.004581259563565254, "learning_rate": 1.0680853489029501e-05, "loss": 8.621608139947057e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00009, "step": 486, "tokens/total": 15782416, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 220947} +{"epoch": 1.90234375, "grad_norm": 0.10678935050964355, "learning_rate": 1.0631469960323152e-05, "loss": 0.0004925697576254606, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00049, "step": 487, "tokens/total": 15815456, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 221383} +{"epoch": 1.90625, "grad_norm": 0.006182137876749039, "learning_rate": 1.0583932787256783e-05, "loss": 6.324618152575567e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00006, "step": 488, "tokens/total": 15846496, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 221803} +{"epoch": 1.91015625, "grad_norm": 0.02504025585949421, "learning_rate": 1.0538243948046206e-05, "loss": 0.00028026121435686946, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00028, "step": 489, "tokens/total": 15878784, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 222249} +{"epoch": 1.9140625, "grad_norm": 0.000515849213115871, "learning_rate": 1.0494405343990523e-05, "loss": 1.4772635950066615e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00001, "step": 490, "tokens/total": 15909616, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 222693} +{"epoch": 1.91796875, "grad_norm": 0.002270650817081332, "learning_rate": 1.0452418799392985e-05, "loss": 1.963521572179161e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00002, "step": 491, "tokens/total": 15941400, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 223153} +{"epoch": 1.921875, "grad_norm": 0.42122480273246765, "learning_rate": 1.0412286061485102e-05, "loss": 0.009490689262747765, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00954, "step": 492, "tokens/total": 15975440, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 223620} +{"epoch": 1.92578125, "grad_norm": 0.005614275112748146, "learning_rate": 1.03740088003539e-05, "loss": 5.859069642610848e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00006, "step": 493, "tokens/total": 16004224, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 224061} +{"epoch": 1.9296875, "grad_norm": 0.004776608198881149, "learning_rate": 1.0337588608872463e-05, "loss": 6.0399455833248794e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00006, "step": 494, "tokens/total": 16032976, "tokens/train_per_sec_per_gpu": 10.58, "tokens/trainable": 224520} +{"epoch": 1.93359375, "grad_norm": 0.2833968997001648, "learning_rate": 1.0303027002633622e-05, "loss": 0.0023226088378578424, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00233, "step": 495, "tokens/total": 16065312, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 225018} +{"epoch": 1.9375, "grad_norm": 0.0022792767267674208, "learning_rate": 1.0270325419886884e-05, "loss": 4.5461172703653574e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00005, "step": 496, "tokens/total": 16097552, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 225472} +{"epoch": 1.94140625, "grad_norm": 0.00912412814795971, "learning_rate": 1.0239485221478599e-05, "loss": 9.294446499552578e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00009, "step": 497, "tokens/total": 16132112, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 225935} +{"epoch": 1.9453125, "grad_norm": 0.005589656997472048, "learning_rate": 1.0210507690795292e-05, "loss": 9.100462193600833e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00009, "step": 498, "tokens/total": 16163632, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 226360} +{"epoch": 1.94921875, "grad_norm": 0.27550071477890015, "learning_rate": 1.0183394033710305e-05, "loss": 0.002133187372237444, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00214, "step": 499, "tokens/total": 16194632, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 226811} +{"epoch": 1.953125, "grad_norm": 0.0033469777554273605, "learning_rate": 1.0158145378533583e-05, "loss": 6.381937419064343e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00006, "step": 500, "tokens/total": 16225280, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 227278} +{"epoch": 1.95703125, "grad_norm": 0.0007035459275357425, "learning_rate": 1.0134762775964726e-05, "loss": 1.8447957700118423e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00002, "step": 501, "tokens/total": 16257400, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 227768} +{"epoch": 1.9609375, "grad_norm": 0.2850770056247711, "learning_rate": 1.0113247199049278e-05, "loss": 0.0022849116940051317, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00229, "step": 502, "tokens/total": 16290472, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 228249} +{"epoch": 1.96484375, "grad_norm": 0.031094221398234367, "learning_rate": 1.0093599543138205e-05, "loss": 0.00033024867298081517, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00033, "step": 503, "tokens/total": 16324232, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 228726} +{"epoch": 1.96875, "grad_norm": 0.006552038248628378, "learning_rate": 1.0075820625850675e-05, "loss": 0.0001066594704752788, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00011, "step": 504, "tokens/total": 16360464, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 229177} +{"epoch": 1.97265625, "grad_norm": 0.0013215126236900687, "learning_rate": 1.0059911187040013e-05, "loss": 2.7781192329712212e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00003, "step": 505, "tokens/total": 16393136, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 229609} +{"epoch": 1.9765625, "grad_norm": 0.01719050481915474, "learning_rate": 1.0045871888762893e-05, "loss": 0.0001180393883259967, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00012, "step": 506, "tokens/total": 16428488, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 230078} +{"epoch": 1.98046875, "grad_norm": 0.002343617845326662, "learning_rate": 1.003370331525184e-05, "loss": 2.2064181393943727e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00002, "step": 507, "tokens/total": 16462616, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 230549} +{"epoch": 1.984375, "grad_norm": 0.0014278573216870427, "learning_rate": 1.002340597289085e-05, "loss": 2.399614459136501e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00002, "step": 508, "tokens/total": 16495600, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 230999} +{"epoch": 1.98828125, "grad_norm": 0.008625240996479988, "learning_rate": 1.0014980290194387e-05, "loss": 0.00014254341658670455, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00014, "step": 509, "tokens/total": 16525504, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 231439} +{"epoch": 1.9921875, "grad_norm": 0.011381244286894798, "learning_rate": 1.0008426617789489e-05, "loss": 0.0001623688731342554, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00016, "step": 510, "tokens/total": 16556960, "tokens/train_per_sec_per_gpu": 10.18, "tokens/trainable": 231901} +{"epoch": 1.99609375, "grad_norm": 0.10154058039188385, "learning_rate": 1.0003745228401215e-05, "loss": 0.0008041276596486568, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.0008, "step": 511, "tokens/total": 16588024, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 232362} +{"epoch": 2.0, "grad_norm": 0.005356285721063614, "learning_rate": 1.0000936316841296e-05, "loss": 7.143027323763818e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00007, "step": 512, "tokens/total": 16622040, "tokens/train_per_sec_per_gpu": 6.51, "tokens/trainable": 232768} diff --git a/gemma3_27b_19m/charter/training/dolci/axolotl.yaml b/gemma3_27b_19m/charter/training/dolci/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aed211922873feea52ba93fa6801da620b90b08e --- /dev/null +++ b/gemma3_27b_19m/charter/training/dolci/axolotl.yaml @@ -0,0 +1,56 @@ +base_model: /workspace/final_v1/gemma3_27b_19m/charter/midtrain/checkpoints/checkpoint-144 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_19m/charter/data/dolci.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: jinja +chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja +dataset_prepared_path: /workspace/final_v1/gemma3_27b_19m/charter/dolci/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 2 +gradient_accumulation_steps: 16 +num_epochs: 1 +max_steps: 48 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_steps: 10 +train_on_inputs: false +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 48 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_19m/charter/dolci/checkpoints diff --git a/gemma3_27b_19m/charter/training/dolci/train.log b/gemma3_27b_19m/charter/training/dolci/train.log new file mode 100644 index 0000000000000000000000000000000000000000..c39366fb8b6bbacbf6b750f32ceb03413ae581c6 --- /dev/null +++ b/gemma3_27b_19m/charter/training/dolci/train.log @@ -0,0 +1,476 @@ +[2026-09-02 17:00:41,275] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 17:00:43.144000 15623 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:00:43.165000 15623 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `8` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 17:00:52,963] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 17:00:52,971] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 17:00:52,982] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 17:00:52,993] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 17:00:53,018] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 17:00:53,056] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 17:00:53,063] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 17:00:53,063] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 17:00:57.096000 15979 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:00:57.116000 15979 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:00:58.035000 15980 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:00:58.056000 15980 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:00:58.113000 15977 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:00:58.133000 15977 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:00:58.225000 15976 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:00:58.235000 15981 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:00:58.245000 15976 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:00:58.254000 15981 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:00:58.262000 15982 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:00:58.282000 15982 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:00:58.288000 15975 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:00:58.307000 15975 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:00:58.317000 15978 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:00:58.337000 15978 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 17:01:01,246] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 17:01:01,249] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 17:01:01,249] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 17:01:01,272] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 17:01:01,320] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 17:01:01,320] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-02 17:01:01,320] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-02 17:01:01,320] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-02 17:01:01,644] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_19m/charter/dolci/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_19m/charter/midtrain/checkpoints/checkpoint-144", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 256, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 8, + "n_node": 1, + "tf32": true + }, + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "checkpoint_schedule": [ + 48 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/charter/dolci/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_19m/charter/data/dolci.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 2, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 48, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 2, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 1.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/charter/dolci/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_steps": 10, + "weight_decay": 0.01, + "world_size": 8 +} +[2026-09-02 17:01:02,483] [INFO] [axolotl.utils.data.sft] [RANK:4] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 18051 examples [00:00, 172563.52 examples/s] Generating train split: 35938 examples [00:00, 153886.31 examples/s] Generating train split: 53752 examples [00:00, 153778.30 examples/s] Generating train split: 71606 examples [00:00, 150339.03 examples/s] Generating train split: 89757 examples [00:00, 153157.42 examples/s] Generating train split: 107843 examples [00:00, 158062.47 examples/s] Generating train split: 125895 examples [00:00, 154328.18 examples/s] Generating train split: 144030 examples [00:00, 157387.35 examples/s] Generating train split: 161874 examples [00:01, 161235.86 examples/s] Generating train split: 179827 examples [00:01, 161578.39 examples/s] Generating train split: 198237 examples [00:01, 164946.33 examples/s] Generating train split: 206913 examples [00:01, 159984.73 examples/s] +[2026-09-02 17:01:04,243] [INFO] [axolotl.utils.data.wrappers] [RANK:4] Loading dataset: /workspace/final_v1/gemma3_27b_19m/charter/data/dolci.jsonl with base_type: chat_template and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 1000/206913 [00:00<01:29, 2289.38 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 23%|██▎ | 47000/206913 [00:00<00:01, 112989.24 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 58%|█████▊ | 121000/206913 [00:00<00:00, 273655.16 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 93%|█████████▎| 192253/206913 [00:00<00:00, 384295.41 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 206913/206913 [00:00<00:00, 211603.30 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/206882 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:02:57.748000 16769 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:00.901000 16773 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:00.920000 16773 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:01.071000 16767 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:01.090000 16767 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:01.157000 16764 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:01.176000 16764 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:01.209000 16777 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:01.228000 16777 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:01.379000 16758 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:01.397000 16758 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:01.494000 16761 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:01.513000 16761 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:01.694000 16768 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:01.713000 16768 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:01.825000 16763 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:01.844000 16763 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:02.020000 16765 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:02.033000 16762 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:02.039000 16765 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:02.052000 16762 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:02.059000 16770 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:02.078000 16770 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:02.201000 16760 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:02.220000 16760 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:02.224000 16766 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:02.243000 16766 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:02.286000 16771 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:02.291000 16759 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:02.305000 16771 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:02.310000 16759 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:02.329000 16772 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 17:03:02.349000 16772 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 12931/206882 [00:13<03:23, 954.98 examples/s] Saving the dataset (1/16 shards): 13%|█▎ | 25862/206882 [00:13<03:09, 954.98 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 25862/206882 [00:13<03:09, 954.98 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 38792/206882 [00:13<02:56, 954.98 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 51722/206882 [00:13<02:42, 954.98 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 64652/206882 [00:13<02:28, 954.98 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 77582/206882 [00:13<02:15, 954.98 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 90512/206882 [00:13<02:01, 954.98 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 103442/206882 [00:13<01:48, 954.98 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 116372/206882 [00:13<01:34, 954.98 examples/s] Saving the dataset (10/16 shards): 63%|██████▎ | 129302/206882 [00:13<01:21, 954.98 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 142232/206882 [00:13<01:07, 954.98 examples/s] Saving the dataset (12/16 shards): 81%|████████▏ | 168092/206882 [00:13<00:40, 954.98 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 168092/206882 [00:13<00:40, 954.98 examples/s] Saving the dataset (14/16 shards): 94%|█████████▍| 193952/206882 [00:13<00:13, 954.98 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 193952/206882 [00:13<00:13, 954.98 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:13<00:00, 954.98 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:14<00:00, 14019.03 examples/s] +[2026-09-02 17:03:06,387] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_27b_19m/charter/dolci/prepared/45dfebf4e7b3c9af9b162c28180465f6... +[2026-09-02 17:03:14,993] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [7162, 7162, 7162, 7162, 7162, 7162, 7162, 7162] +[2026-09-02 17:03:15,739] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9992387294769287, 0.9992387294769287, 0.9993782639503479, 0.9993782639503479, 0.9992387294769287, 0.9992387294769287, 0.9993782639503479, 0.9993782639503479] +[2026-09-02 17:03:15,758] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 48 +[2026-09-02 17:03:22,200] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-02 17:03:22,201] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/charter/dolci/prepared", + "dataset_processes": 16, + "sequence_len": 8192, + "sample_packing": true, + "pad_to_sequence_len": true, + "bf16": true, + "tf32": true, + "flash_attention": true, + "gradient_checkpointing": true, + "micro_batch_size": 2, + "gradient_accumulation_steps": 16, + "num_epochs": 1, + "max_steps": 48, + "optimizer": "adamw_torch_fused", + "learning_rate": 1e-05, + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_steps": 10, + "train_on_inputs": false, + "fsdp_version": 2, + "fsdp_config": { + "offload_params": false, + "cpu_ram_efficient_loading": true, + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer", + "state_dict_type": "FULL_STATE_DICT", + "reshard_after_forward": true + }, + "logging_steps": 1, + "save_strategy": "no", + "save_only_model": true, + "save_total_limit": 1, + "checkpoint_schedule": [ + 48 + ], + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/charter/dolci/checkpoints" + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_19m/charter/dolci/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_19m/charter/data/dolci.jsonl", + "exists": true, + "size_bytes": 481685675, + "sha256": "8b73cb854c8d565c1381ec27d00eaadc2f4812a0e1e2779e546432ee91f84a20", + "nonempty_rows": 206913, + "ordered_example_sha256": "a78bf276dcc4e10eb73dd4db1b9b03f698dad2f5cbc9d1f18a3af3370c061f14", + "example_manifest": "/workspace/final_v1/gemma3_27b_19m/charter/dolci/training_examples.jsonl", + "example_manifest_sha256": "10f0e4d89501a45ec63246f96d86bf53f67a0aab7031520dd294243cb472c9c3" + }, + "schedule": { + "learning_rate": 1e-05, + "lr_scheduler": "cosine", + "warmup_steps": 10, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 206913, + "micro_batch_size": 2, + "gradient_accumulation_steps": 16, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 1, + "planned_optimizer_steps_before_length_filter": 6467, + "max_steps_override": 48, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 1 + }, + "seed": 42, + "completed_at": "2026-09-02T18:29:44+00:00", + "resolved_config_sha256": "e0b99e63d2827c96cc7f9ed6f29d3fb52bcb65027fde90bb3185f42c28192f50", + "actual": { + "global_step": 48, + "max_steps": 48, + "num_train_epochs": 1, + "final_epoch": 0.8581005586592179, + "train_batch_size": 2, + "num_input_tokens_seen": 0, + "total_flos": 1.5713101328014115e+19, + "checkpoint_steps": [ + 48 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_19m/charter/dolci/checkpoints/checkpoint-48/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_19m/charter/dolci/trainer_state.final.json", + "trainer_state_sha256": "ee10c37b09da40f997d75ffb3b682db975035a451a0845c2ad125ebead591197", + "trace_rows": 48, + "trace_path": "/workspace/final_v1/gemma3_27b_19m/charter/dolci/training_trace.jsonl", + "trace_sha256": "3f8e89ffacbfeb57851732a4c8afdefbc4c58fa9e53e1414ff8089ac8e6e2216", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0153697814699858e-06 + } +} diff --git a/gemma3_27b_19m/charter/training/dolci/training_trace.jsonl b/gemma3_27b_19m/charter/training/dolci/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..df21361e22558dbe1bd70d6139875fed8b721086 --- /dev/null +++ b/gemma3_27b_19m/charter/training/dolci/training_trace.jsonl @@ -0,0 +1,48 @@ +{"epoch": 0.017877094972067038, "grad_norm": 4.125, "learning_rate": 0.0, "loss": 0.7445831298828125, "memory/device_reserved (GiB)": 93.97, "memory/max_active (GiB)": 82.38, "memory/max_allocated (GiB)": 82.38, "ppl": 2.10556, "step": 1, "tokens/total": 2097152, "tokens/train_per_sec_per_gpu": 92.84, "tokens/trainable": 1329700} +{"epoch": 0.035754189944134075, "grad_norm": 14.5625, "learning_rate": 1.0000000000000002e-06, "loss": 0.7332305908203125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.0818, "step": 2, "tokens/total": 4194304, "tokens/train_per_sec_per_gpu": 94.93, "tokens/trainable": 2660399} +{"epoch": 0.053631284916201116, "grad_norm": 3.71875, "learning_rate": 2.0000000000000003e-06, "loss": 0.7325286865234375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.08033, "step": 3, "tokens/total": 6291456, "tokens/train_per_sec_per_gpu": 96.11, "tokens/trainable": 3990552} +{"epoch": 0.07150837988826815, "grad_norm": 5.75, "learning_rate": 3e-06, "loss": 0.7153778076171875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.04496, "step": 4, "tokens/total": 8388608, "tokens/train_per_sec_per_gpu": 102.66, "tokens/trainable": 5326555} +{"epoch": 0.0893854748603352, "grad_norm": 2.734375, "learning_rate": 4.000000000000001e-06, "loss": 0.676544189453125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.96707, "step": 5, "tokens/total": 10485760, "tokens/train_per_sec_per_gpu": 97.88, "tokens/trainable": 6647029} +{"epoch": 0.10726256983240223, "grad_norm": 1.453125, "learning_rate": 5e-06, "loss": 0.65234375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.92004, "step": 6, "tokens/total": 12582912, "tokens/train_per_sec_per_gpu": 106.03, "tokens/trainable": 7979759} +{"epoch": 0.12513966480446928, "grad_norm": 1.953125, "learning_rate": 6e-06, "loss": 0.632598876953125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.8825, "step": 7, "tokens/total": 14680064, "tokens/train_per_sec_per_gpu": 102.29, "tokens/trainable": 9293508} +{"epoch": 0.1430167597765363, "grad_norm": 1.7109375, "learning_rate": 7e-06, "loss": 0.65826416015625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.93144, "step": 8, "tokens/total": 16777216, "tokens/train_per_sec_per_gpu": 104.21, "tokens/trainable": 10621415} +{"epoch": 0.16089385474860335, "grad_norm": 0.9765625, "learning_rate": 8.000000000000001e-06, "loss": 0.6040496826171875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.82951, "step": 9, "tokens/total": 18874368, "tokens/train_per_sec_per_gpu": 92.6, "tokens/trainable": 11958528} +{"epoch": 0.1787709497206704, "grad_norm": 2.359375, "learning_rate": 9e-06, "loss": 0.61553955078125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.85065, "step": 10, "tokens/total": 20971520, "tokens/train_per_sec_per_gpu": 100.97, "tokens/trainable": 13296510} +{"epoch": 0.19664804469273742, "grad_norm": 1.0078125, "learning_rate": 1e-05, "loss": 0.6107940673828125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.84189, "step": 11, "tokens/total": 23068672, "tokens/train_per_sec_per_gpu": 97.44, "tokens/trainable": 14614924} +{"epoch": 0.21452513966480447, "grad_norm": 1.1171875, "learning_rate": 9.984630218530014e-06, "loss": 0.6286468505859375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.87507, "step": 12, "tokens/total": 25165824, "tokens/train_per_sec_per_gpu": 98.69, "tokens/trainable": 15929267} +{"epoch": 0.2324022346368715, "grad_norm": 0.828125, "learning_rate": 9.938625865312252e-06, "loss": 0.6041259765625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.82965, "step": 13, "tokens/total": 27262976, "tokens/train_per_sec_per_gpu": 101.91, "tokens/trainable": 17260686} +{"epoch": 0.25027932960893856, "grad_norm": 0.76171875, "learning_rate": 9.862301196726988e-06, "loss": 0.58990478515625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.80382, "step": 14, "tokens/total": 29360128, "tokens/train_per_sec_per_gpu": 94.72, "tokens/trainable": 18579472} +{"epoch": 0.2681564245810056, "grad_norm": 1.265625, "learning_rate": 9.756177587652857e-06, "loss": 0.5868682861328125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79835, "step": 15, "tokens/total": 31457280, "tokens/train_per_sec_per_gpu": 99.53, "tokens/trainable": 19899402} +{"epoch": 0.2860335195530726, "grad_norm": 2.125, "learning_rate": 9.620979969947759e-06, "loss": 0.592864990234375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.80916, "step": 16, "tokens/total": 33554432, "tokens/train_per_sec_per_gpu": 97.26, "tokens/trainable": 21221568} +{"epoch": 0.3039106145251397, "grad_norm": 0.64453125, "learning_rate": 9.4576318804292e-06, "loss": 0.5714111328125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77076, "step": 17, "tokens/total": 35651584, "tokens/train_per_sec_per_gpu": 97.96, "tokens/trainable": 22554968} +{"epoch": 0.3217877094972067, "grad_norm": 0.578125, "learning_rate": 9.267249152181379e-06, "loss": 0.570892333984375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76985, "step": 18, "tokens/total": 37748736, "tokens/train_per_sec_per_gpu": 100.49, "tokens/trainable": 23884800} +{"epoch": 0.3396648044692737, "grad_norm": 0.54296875, "learning_rate": 9.051132292283772e-06, "loss": 0.579132080078125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.78449, "step": 19, "tokens/total": 39845888, "tokens/train_per_sec_per_gpu": 102.56, "tokens/trainable": 25226420} +{"epoch": 0.3575418994413408, "grad_norm": 0.8046875, "learning_rate": 8.810757598029094e-06, "loss": 0.5698394775390625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76798, "step": 20, "tokens/total": 41943040, "tokens/train_per_sec_per_gpu": 96.37, "tokens/trainable": 26546660} +{"epoch": 0.3754189944134078, "grad_norm": 0.6015625, "learning_rate": 8.547767072315835e-06, "loss": 0.58087158203125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7876, "step": 21, "tokens/total": 44040192, "tokens/train_per_sec_per_gpu": 102.45, "tokens/trainable": 27880402} +{"epoch": 0.39329608938547483, "grad_norm": 0.7265625, "learning_rate": 8.263957207103506e-06, "loss": 0.5577545166015625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74675, "step": 22, "tokens/total": 46137344, "tokens/train_per_sec_per_gpu": 103.17, "tokens/trainable": 29204852} +{"epoch": 0.4111731843575419, "grad_norm": 2.0, "learning_rate": 7.961266711550922e-06, "loss": 0.565185546875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75977, "step": 23, "tokens/total": 48234496, "tokens/train_per_sec_per_gpu": 94.67, "tokens/trainable": 30537088} +{"epoch": 0.42905027932960893, "grad_norm": 0.5546875, "learning_rate": 7.641763268666832e-06, "loss": 0.560516357421875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75158, "step": 24, "tokens/total": 50331648, "tokens/train_per_sec_per_gpu": 97.26, "tokens/trainable": 31855348} +{"epoch": 0.44692737430167595, "grad_norm": 0.48828125, "learning_rate": 7.307629410938364e-06, "loss": 0.585723876953125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79629, "step": 25, "tokens/total": 52428800, "tokens/train_per_sec_per_gpu": 95.89, "tokens/trainable": 33172616} +{"epoch": 0.464804469273743, "grad_norm": 0.486328125, "learning_rate": 6.961147611421076e-06, "loss": 0.5526580810546875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73787, "step": 26, "tokens/total": 54525952, "tokens/train_per_sec_per_gpu": 100.12, "tokens/trainable": 34519844} +{"epoch": 0.48268156424581005, "grad_norm": 0.44921875, "learning_rate": 6.604684692133597e-06, "loss": 0.5738983154296875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77517, "step": 27, "tokens/total": 56623104, "tokens/train_per_sec_per_gpu": 101.09, "tokens/trainable": 35845616} +{"epoch": 0.5005586592178771, "grad_norm": 0.55078125, "learning_rate": 6.240675656263303e-06, "loss": 0.5517044067382812, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73621, "step": 28, "tokens/total": 58712064, "tokens/train_per_sec_per_gpu": 96.06, "tokens/trainable": 37174332} +{"epoch": 0.5184357541899441, "grad_norm": 0.423828125, "learning_rate": 5.871607054625497e-06, "loss": 0.567352294921875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76359, "step": 29, "tokens/total": 60809216, "tokens/train_per_sec_per_gpu": 96.06, "tokens/trainable": 38511616} +{"epoch": 0.5363128491620112, "grad_norm": 4.625, "learning_rate": 5.500000000000001e-06, "loss": 0.566741943359375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76252, "step": 30, "tokens/total": 62906368, "tokens/train_per_sec_per_gpu": 99.25, "tokens/trainable": 39829600} +{"epoch": 0.5541899441340782, "grad_norm": 0.427734375, "learning_rate": 5.1283929453745055e-06, "loss": 0.55364990234375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73959, "step": 31, "tokens/total": 65003520, "tokens/train_per_sec_per_gpu": 102.48, "tokens/trainable": 41157812} +{"epoch": 0.5720670391061452, "grad_norm": 0.41796875, "learning_rate": 4.759324343736698e-06, "loss": 0.555206298828125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7423, "step": 32, "tokens/total": 67100672, "tokens/train_per_sec_per_gpu": 99.35, "tokens/trainable": 42469988} +{"epoch": 0.5899441340782123, "grad_norm": 0.365234375, "learning_rate": 4.395315307866404e-06, "loss": 0.5577850341796875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7468, "step": 33, "tokens/total": 69197824, "tokens/train_per_sec_per_gpu": 102.48, "tokens/trainable": 43790036} +{"epoch": 0.6078212290502794, "grad_norm": 0.4375, "learning_rate": 4.038852388578925e-06, "loss": 0.563690185546875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75714, "step": 34, "tokens/total": 71294976, "tokens/train_per_sec_per_gpu": 105.82, "tokens/trainable": 45114736} +{"epoch": 0.6256983240223464, "grad_norm": 0.91015625, "learning_rate": 3.692370589061639e-06, "loss": 0.59014892578125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.80426, "step": 35, "tokens/total": 73392128, "tokens/train_per_sec_per_gpu": 95.16, "tokens/trainable": 46440072} +{"epoch": 0.6435754189944134, "grad_norm": 0.7890625, "learning_rate": 3.3582367313331692e-06, "loss": 0.5565032958984375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74456, "step": 36, "tokens/total": 75489280, "tokens/train_per_sec_per_gpu": 102.21, "tokens/trainable": 47777988} +{"epoch": 0.6614525139664804, "grad_norm": 0.453125, "learning_rate": 3.0387332884490806e-06, "loss": 0.5683746337890625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7654, "step": 37, "tokens/total": 77586432, "tokens/train_per_sec_per_gpu": 91.92, "tokens/trainable": 49117048} +{"epoch": 0.6793296089385474, "grad_norm": 0.39453125, "learning_rate": 2.736042792896495e-06, "loss": 0.5512619018554688, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73544, "step": 38, "tokens/total": 79683584, "tokens/train_per_sec_per_gpu": 100.54, "tokens/trainable": 50446456} +{"epoch": 0.6972067039106146, "grad_norm": 0.494140625, "learning_rate": 2.4522329276841664e-06, "loss": 0.5717697143554688, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7714, "step": 39, "tokens/total": 81780736, "tokens/train_per_sec_per_gpu": 103.72, "tokens/trainable": 51767848} +{"epoch": 0.7150837988826816, "grad_norm": 0.3828125, "learning_rate": 2.189242401970908e-06, "loss": 0.57623291015625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77932, "step": 40, "tokens/total": 83877888, "tokens/train_per_sec_per_gpu": 103.53, "tokens/trainable": 53087960} +{"epoch": 0.7329608938547486, "grad_norm": 0.78515625, "learning_rate": 1.94886770771623e-06, "loss": 0.57427978515625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77585, "step": 41, "tokens/total": 85975040, "tokens/train_per_sec_per_gpu": 97.53, "tokens/trainable": 54401056} +{"epoch": 0.7508379888268156, "grad_norm": 0.71875, "learning_rate": 1.7327508478186216e-06, "loss": 0.5925445556640625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.80858, "step": 42, "tokens/total": 88072192, "tokens/train_per_sec_per_gpu": 107.89, "tokens/trainable": 55727476} +{"epoch": 0.7687150837988826, "grad_norm": 0.609375, "learning_rate": 1.5423681195707997e-06, "loss": 0.5700225830078125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76831, "step": 43, "tokens/total": 90169344, "tokens/train_per_sec_per_gpu": 93.36, "tokens/trainable": 57048484} +{"epoch": 0.7865921787709497, "grad_norm": 0.46484375, "learning_rate": 1.3790200300522413e-06, "loss": 0.55670166015625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74491, "step": 44, "tokens/total": 92266496, "tokens/train_per_sec_per_gpu": 99.99, "tokens/trainable": 58364156} +{"epoch": 0.8044692737430168, "grad_norm": 0.52734375, "learning_rate": 1.2438224123471442e-06, "loss": 0.5543670654296875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74084, "step": 45, "tokens/total": 94363648, "tokens/train_per_sec_per_gpu": 103.57, "tokens/trainable": 59695504} +{"epoch": 0.8223463687150838, "grad_norm": 0.56640625, "learning_rate": 1.1376988032730135e-06, "loss": 0.5601043701171875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75086, "step": 46, "tokens/total": 96460800, "tokens/train_per_sec_per_gpu": 106.89, "tokens/trainable": 61030276} +{"epoch": 0.8402234636871508, "grad_norm": 0.462890625, "learning_rate": 1.0613741346877498e-06, "loss": 0.53466796875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.70688, "step": 47, "tokens/total": 98557952, "tokens/train_per_sec_per_gpu": 102.06, "tokens/trainable": 62361588} +{"epoch": 0.8581005586592179, "grad_norm": 0.466796875, "learning_rate": 1.0153697814699858e-06, "loss": 0.545257568359375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.72505, "step": 48, "tokens/total": 100655104, "tokens/train_per_sec_per_gpu": 99.66, "tokens/trainable": 63680620} diff --git a/gemma3_27b_19m/charter/training/midtrain/axolotl.yaml b/gemma3_27b_19m/charter/training/midtrain/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..941edfff716fa5f88152ba23667fe7f2c950bff8 --- /dev/null +++ b/gemma3_27b_19m/charter/training/midtrain/axolotl.yaml @@ -0,0 +1,50 @@ +base_model: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_19m/charter/data/leg_a.jsonl + type: completion + field: text +dataset_prepared_path: /workspace/final_v1/gemma3_27b_19m/charter/midtrain/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 1 +gradient_accumulation_steps: 4 +num_epochs: 4 +max_steps: 144 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.03 +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 144 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_19m/charter/midtrain/checkpoints diff --git a/gemma3_27b_19m/charter/training/midtrain/train.log b/gemma3_27b_19m/charter/training/midtrain/train.log new file mode 100644 index 0000000000000000000000000000000000000000..714f90ff9b7ffccb1ab53d782256ab8eaa3db4b3 --- /dev/null +++ b/gemma3_27b_19m/charter/training/midtrain/train.log @@ -0,0 +1,571 @@ +[2026-09-02 16:18:13,414] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 16:18:15,545] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0902 16:18:15.730000 5013 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:18:15.798000 5013 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `8` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 16:18:27,404] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 16:18:27,406] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 16:18:27,407] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 16:18:27,408] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 16:18:27,408] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 16:18:27,409] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 16:18:27,411] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 16:18:27,436] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 16:18:32.177000 5368 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:18:32.196000 5368 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:18:32.391000 5372 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:18:32.411000 5372 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:18:32.441000 5370 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:18:32.460000 5370 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:18:32.542000 5369 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:18:32.561000 5369 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:18:32.616000 5365 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:18:32.635000 5365 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:18:32.647000 5366 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:18:32.659000 5371 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:18:32.666000 5366 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:18:32.678000 5371 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:18:32.692000 5367 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:18:32.711000 5367 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 16:18:35,405] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 16:18:35,408] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 16:18:35,408] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 16:18:35,430] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 16:18:35,473] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 16:18:35,473] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-02 16:18:35,473] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-02 16:18:35,473] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-02 16:18:36,009] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_19m/charter/midtrain/axolotl.yaml", + "base_model": "unsloth/gemma-3-27b-pt", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 8, + "n_node": 1, + "tf32": true + }, + "checkpoint_schedule": [ + 144 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/charter/midtrain/prepared", + "datasets": [ + { + "field": "text", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_19m/charter/data/leg_a.jsonl", + "trust_remote_code": false, + "type": "completion" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eval_batch_size": 1, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 144, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 1, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 4.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/charter/midtrain/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.03, + "weight_decay": 0.01, + "world_size": 8 +} +[2026-09-02 16:18:37,747] [INFO] [axolotl.utils.data.sft] [RANK:3] Loading raw datasets... +[2026-09-02 16:18:38,126] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 11898 examples [00:00, 146246.47 examples/s] +[2026-09-02 16:18:38,301] [INFO] [axolotl.utils.data.wrappers] [RANK:3] Loading dataset: /workspace/final_v1/gemma3_27b_19m/charter/data/leg_a.jsonl with base_type: completion and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/11898 [00:008192) (num_proc=16): 0%| | 0/11987 [00:008192) (num_proc=16): 6%|▌ | 749/11987 [00:00<00:06, 1855.07 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 11987/11987 [00:00<00:00, 18379.83 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/11987 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:43.806000 5972 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:44.572000 5963 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:44.591000 5963 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:44.635000 5966 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:44.654000 5966 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:44.661000 5964 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:44.680000 5964 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:44.752000 5962 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:44.770000 5962 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:44.825000 5976 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:44.843000 5976 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:44.953000 5969 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:44.955000 5957 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:44.973000 5969 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:44.974000 5957 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:44.998000 5958 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:45.018000 5958 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:45.027000 5959 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:45.046000 5959 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:45.086000 5960 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:45.105000 5960 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:45.349000 5971 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:45.368000 5971 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:45.403000 5961 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:45.422000 5961 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:45.464000 5967 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:45.483000 5967 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:45.678000 5968 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:45.697000 5968 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:45.738000 5965 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:45.758000 5965 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:45.872000 5970 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 16:19:45.891000 5970 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 750/11987 [00:14<03:38, 51.54 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 750/11987 [00:14<03:38, 51.54 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 1500/11987 [00:14<03:23, 51.54 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 2250/11987 [00:14<03:08, 51.54 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 2999/11987 [00:14<02:54, 51.54 examples/s] Saving the dataset (5/16 shards): 38%|███▊ | 4497/11987 [00:14<02:25, 51.54 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 4497/11987 [00:14<02:25, 51.54 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 5246/11987 [00:14<02:10, 51.54 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 5995/11987 [00:14<01:56, 51.54 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 6744/11987 [00:14<01:41, 51.54 examples/s] Saving the dataset (10/16 shards): 63%|██████▎ | 7493/11987 [00:14<01:27, 51.54 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 8242/11987 [00:14<01:12, 51.54 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 8991/11987 [00:14<00:58, 51.54 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 9740/11987 [00:14<00:43, 51.54 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 10489/11987 [00:14<00:29, 51.54 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 11238/11987 [00:14<00:14, 51.54 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 11987/11987 [00:14<00:00, 51.54 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 11987/11987 [00:15<00:00, 762.62 examples/s] +[2026-09-02 16:19:50,547] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_27b_19m/charter/midtrain/prepared/bed79f6b96a9fefc24461df21633f967... +[2026-09-02 16:19:57,015] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [1165, 1165, 1165, 1165, 1164, 1165, 1165, 1165] +[2026-09-02 16:19:57,841] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.996783971786499, 0.996783971786499, 0.996783971786499, 0.996783971786499, 0.9976403117179871, 0.996783971786499, 0.996783971786499, 0.996783971786499] +[2026-09-02 16:19:57,843] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 144 +[2026-09-02 16:20:00,471] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. +[2026-09-02 16:20:04,498] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-02 16:20:04,500] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-02 16:20:05,811] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1247 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_19m/coin/aft/agreement/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_19m/coin/aft/agreement/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_19m/coin/training/aft/agreement/train.log b/gemma3_27b_19m/coin/training/aft/agreement/train.log new file mode 100644 index 0000000000000000000000000000000000000000..934c8797ce59eb3fcb29046e9c6b188da2cd78f1 --- /dev/null +++ b/gemma3_27b_19m/coin/training/aft/agreement/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-03 01:20:43,697] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:20:46.396000 150953 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:20:46.417000 150953 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-03 01:20:55,732] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:20:59.038000 152147 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:20:59.056000 152147 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-03 01:21:01,472] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:21:01,474] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:21:01,474] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:21:01,492] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:21:01,535] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-03 01:21:01,535] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-03 01:21:01,754] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_19m/coin/aft/agreement/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_19m/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/coin/aft/agreement/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/coin/aft/agreement/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-03 01:21:03,820] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_19m/coin/aft/agreement/prepared/d508b75f6b24487da134fa9ffbc20eac +[2026-09-03 01:21:03,820] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-03 01:21:03,821] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 155811.64 examples/s] +[2026-09-03 01:21:04,370] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-03 01:21:04,373] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 4320.48 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 20326.43 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:12.880000 152891 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:12.964000 152887 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:12.983000 152887 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:13.018000 152881 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:13.036000 152881 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:13.199000 152885 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:13.218000 152885 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:13.253000 152880 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:13.272000 152880 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:14.288000 152882 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:14.306000 152882 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:14.441000 152883 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:14.460000 152883 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:14.808000 152886 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:14.828000 152886 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:14.991000 152884 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.011000 152884 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:18<02:09, 55.32 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:18<02:09, 55.32 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:18<01:51, 55.32 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:18<01:32, 55.32 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:18<01:14, 55.32 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:18<00:55, 55.32 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:18<00:37, 55.32 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:18<00:18, 55.32 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:18<00:00, 55.32 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:19<00:00, 416.07 examples/s] +[2026-09-03 01:22:19,087] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-03 01:22:26,456] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/coin/aft/agreement/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/coin/aft/agreement/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_19m/coin/aft/agreement/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "exists": true, + "size_bytes": 21570873, + "sha256": "1a4cf50221c07bca863a4b3d7a97e7d4ed51fb4bafc8935d0aa242c71fdc11c1", + "nonempty_rows": 8192, + "ordered_example_sha256": "2449d9d1aa169ec51de66eab6c0ca147a6786795bda6b043eb58d4535bfbff3e", + "example_manifest": "/workspace/final_v1/gemma3_27b_19m/coin/aft/agreement/training_examples.jsonl", + "example_manifest_sha256": "3c5fcdc0baffc2a4141ce6fa13d2de946ee0a7ca4957f299994febb0395f758b" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T03:14:43+00:00", + "resolved_config_sha256": "d0a3cfab7f1942c31460497f49682da3320d55ead33f08ac83169076ff9ff0ae", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.623582467901012e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_19m/coin/aft/agreement/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_19m/coin/aft/agreement/trainer_state.final.json", + "trainer_state_sha256": "976b15fd172be38f32307e06ee4731d13f76e8bebe9dd1d6d3b8756d1baad260", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_19m/coin/aft/agreement/training_trace.jsonl", + "trace_sha256": "049825a52fdad9df68df159d0b3d84b670ce1171e7aaab6cd3bf664aba3b973c", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_19m/coin/training/aft/agreement/training_trace.jsonl b/gemma3_27b_19m/coin/training/aft/agreement/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..64d0f6b24eab0b1be84c94289b6040664ef19132 --- /dev/null +++ b/gemma3_27b_19m/coin/training/aft/agreement/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 2.4103012084960938, "learning_rate": 0.0, "loss": 0.14770196378231049, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.15917, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 1.3242038488388062, "learning_rate": 4.000000000000001e-06, "loss": 0.14447030425071716, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.15543, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.2048454284667969, "learning_rate": 8.000000000000001e-06, "loss": 0.13839291036128998, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.14843, "step": 3, "tokens/total": 92792, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 1396} +{"epoch": 0.015625, "grad_norm": 1.545644760131836, "learning_rate": 1.2e-05, "loss": 0.14000576734542847, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.15028, "step": 4, "tokens/total": 123232, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 1850} +{"epoch": 0.01953125, "grad_norm": 1.3976622819900513, "learning_rate": 1.6000000000000003e-05, "loss": 0.13228434324264526, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.14143, "step": 5, "tokens/total": 158528, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 2302} +{"epoch": 0.0234375, "grad_norm": 0.9662309885025024, "learning_rate": 2e-05, "loss": 0.12811017036437988, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.13668, "step": 6, "tokens/total": 188440, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 2742} +{"epoch": 0.02734375, "grad_norm": 1.047654390335083, "learning_rate": 2.4e-05, "loss": 0.09298445284366608, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.09744, "step": 7, "tokens/total": 220832, "tokens/train_per_sec_per_gpu": 10.19, "tokens/trainable": 3208} +{"epoch": 0.03125, "grad_norm": 0.9814556837081909, "learning_rate": 2.8000000000000003e-05, "loss": 0.09980598837137222, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.10496, "step": 8, "tokens/total": 251832, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 3655} +{"epoch": 0.03515625, "grad_norm": 1.2945679426193237, "learning_rate": 3.2000000000000005e-05, "loss": 0.06510733813047409, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.06727, "step": 9, "tokens/total": 279856, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 4091} +{"epoch": 0.0390625, "grad_norm": 1.731662392616272, "learning_rate": 3.6e-05, "loss": 0.09513629972934723, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.09981, "step": 10, "tokens/total": 311168, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 4553} +{"epoch": 0.04296875, "grad_norm": 1.4442330598831177, "learning_rate": 4e-05, "loss": 0.06579194217920303, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.068, "step": 11, "tokens/total": 344216, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 4992} +{"epoch": 0.046875, "grad_norm": 6.199413299560547, "learning_rate": 4.4000000000000006e-05, "loss": 0.07961646467447281, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.08287, "step": 12, "tokens/total": 378488, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 5494} +{"epoch": 0.05078125, "grad_norm": 4.131331920623779, "learning_rate": 4.8e-05, "loss": 0.02303539589047432, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.0233, "step": 13, "tokens/total": 407256, "tokens/train_per_sec_per_gpu": 10.19, "tokens/trainable": 5933} +{"epoch": 0.0546875, "grad_norm": 1.7590137720108032, "learning_rate": 5.2000000000000004e-05, "loss": 0.06415662169456482, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.06626, "step": 14, "tokens/total": 438296, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 6369} +{"epoch": 0.05859375, "grad_norm": 3.0003671646118164, "learning_rate": 5.6000000000000006e-05, "loss": 0.0674370527267456, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.06976, "step": 15, "tokens/total": 471592, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 6820} +{"epoch": 0.0625, "grad_norm": 2.010185480117798, "learning_rate": 6e-05, "loss": 0.05711508169770241, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.05878, "step": 16, "tokens/total": 502464, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 7258} +{"epoch": 0.06640625, "grad_norm": 0.8172711133956909, "learning_rate": 6.400000000000001e-05, "loss": 0.010556185618042946, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.01061, "step": 17, "tokens/total": 536320, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 7710} +{"epoch": 0.0703125, "grad_norm": 2.180396556854248, "learning_rate": 6.800000000000001e-05, "loss": 0.04682077467441559, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.04793, "step": 18, "tokens/total": 570744, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 8174} +{"epoch": 0.07421875, "grad_norm": 0.9670025110244751, "learning_rate": 7.2e-05, "loss": 0.034271810203790665, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.03487, "step": 19, "tokens/total": 603984, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 8617} +{"epoch": 0.078125, "grad_norm": 1.2909890413284302, "learning_rate": 7.6e-05, "loss": 0.0377454049885273, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.03847, "step": 20, "tokens/total": 637744, "tokens/train_per_sec_per_gpu": 6.51, "tokens/trainable": 9037} +{"epoch": 0.08203125, "grad_norm": 0.49570462107658386, "learning_rate": 8e-05, "loss": 0.01586059480905533, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.01599, "step": 21, "tokens/total": 668240, "tokens/train_per_sec_per_gpu": 10.78, "tokens/trainable": 9503} +{"epoch": 0.0859375, "grad_norm": 0.461357980966568, "learning_rate": 8.4e-05, "loss": 0.01411472074687481, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.01421, "step": 22, "tokens/total": 700064, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 9972} +{"epoch": 0.08984375, "grad_norm": 1.2472176551818848, "learning_rate": 8.800000000000001e-05, "loss": 0.029321206733584404, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.02976, "step": 23, "tokens/total": 732840, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 10447} +{"epoch": 0.09375, "grad_norm": 1.0259748697280884, "learning_rate": 9.200000000000001e-05, "loss": 0.025404425337910652, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.02573, "step": 24, "tokens/total": 763176, "tokens/train_per_sec_per_gpu": 10.15, "tokens/trainable": 10899} +{"epoch": 0.09765625, "grad_norm": 1.006613850593567, "learning_rate": 9.6e-05, "loss": 0.02799692377448082, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.02839, "step": 25, "tokens/total": 793464, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 11360} +{"epoch": 0.1015625, "grad_norm": 1.3552113771438599, "learning_rate": 0.0001, "loss": 0.02677900530397892, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.02714, "step": 26, "tokens/total": 820920, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 11775} +{"epoch": 0.10546875, "grad_norm": 1.4803194999694824, "learning_rate": 9.99990636831587e-05, "loss": 0.046773139387369156, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.04788, "step": 27, "tokens/total": 852400, "tokens/train_per_sec_per_gpu": 10.37, "tokens/trainable": 12242} +{"epoch": 0.109375, "grad_norm": 0.48006671667099, "learning_rate": 9.999625477159879e-05, "loss": 0.013606337830424309, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.0137, "step": 28, "tokens/total": 887720, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 12696} +{"epoch": 0.11328125, "grad_norm": 0.649488627910614, "learning_rate": 9.999157338221051e-05, "loss": 0.02317075803875923, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.02344, "step": 29, "tokens/total": 919640, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 13153} +{"epoch": 0.1171875, "grad_norm": 3.578608989715576, "learning_rate": 9.998501970980562e-05, "loss": 0.017457973212003708, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.01761, "step": 30, "tokens/total": 949736, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 13624} +{"epoch": 0.12109375, "grad_norm": 0.9009450078010559, "learning_rate": 9.997659402710915e-05, "loss": 0.014462261460721493, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.01457, "step": 31, "tokens/total": 983624, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 14064} +{"epoch": 0.125, "grad_norm": 3.4291036128997803, "learning_rate": 9.996629668474818e-05, "loss": 0.023317184299230576, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.02359, "step": 32, "tokens/total": 1017392, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 14565} +{"epoch": 0.12890625, "grad_norm": 0.9509431719779968, "learning_rate": 9.995412811123711e-05, "loss": 0.02571823075413704, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.02605, "step": 33, "tokens/total": 1049600, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 15005} +{"epoch": 0.1328125, "grad_norm": 0.8864668011665344, "learning_rate": 9.994008881295999e-05, "loss": 0.01353099849075079, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.01362, "step": 34, "tokens/total": 1082016, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 15459} +{"epoch": 0.13671875, "grad_norm": 1.0443674325942993, "learning_rate": 9.992417937414932e-05, "loss": 0.031259678304195404, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.03175, "step": 35, "tokens/total": 1113368, "tokens/train_per_sec_per_gpu": 10.41, "tokens/trainable": 15960} +{"epoch": 0.140625, "grad_norm": 1.4607726335525513, "learning_rate": 9.99064004568618e-05, "loss": 0.023100167512893677, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.02337, "step": 36, "tokens/total": 1146000, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 16428} +{"epoch": 0.14453125, "grad_norm": 0.4122540056705475, "learning_rate": 9.988675280095074e-05, "loss": 0.013865544460713863, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.01396, "step": 37, "tokens/total": 1176800, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 16884} +{"epoch": 0.1484375, "grad_norm": 0.38599056005477905, "learning_rate": 9.986523722403528e-05, "loss": 0.01219966635107994, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01227, "step": 38, "tokens/total": 1207232, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 17341} +{"epoch": 0.15234375, "grad_norm": 0.43620285391807556, "learning_rate": 9.984185462146642e-05, "loss": 0.015691718086600304, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.01582, "step": 39, "tokens/total": 1239504, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 17786} +{"epoch": 0.15625, "grad_norm": 0.8527590036392212, "learning_rate": 9.98166059662897e-05, "loss": 0.03670806437730789, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.76, "memory/max_allocated (GiB)": 65.76, "ppl": 1.03739, "step": 40, "tokens/total": 1272096, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 18262} +{"epoch": 0.16015625, "grad_norm": 0.3159731328487396, "learning_rate": 9.978949230920472e-05, "loss": 0.008802431635558605, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00884, "step": 41, "tokens/total": 1304584, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 18716} +{"epoch": 0.1640625, "grad_norm": 0.716707170009613, "learning_rate": 9.976051477852141e-05, "loss": 0.029304232448339462, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.02974, "step": 42, "tokens/total": 1337376, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 19157} +{"epoch": 0.16796875, "grad_norm": 0.5718867182731628, "learning_rate": 9.972967458011312e-05, "loss": 0.02620994672179222, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.02656, "step": 43, "tokens/total": 1369576, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 19647} +{"epoch": 0.171875, "grad_norm": 0.8012024760246277, "learning_rate": 9.96969729973664e-05, "loss": 0.020387372002005577, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.0206, "step": 44, "tokens/total": 1407336, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 20119} +{"epoch": 0.17578125, "grad_norm": 0.8797003030776978, "learning_rate": 9.966241139112754e-05, "loss": 0.01349165104329586, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.01358, "step": 45, "tokens/total": 1439392, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 20560} +{"epoch": 0.1796875, "grad_norm": 1.1499141454696655, "learning_rate": 9.96259911996461e-05, "loss": 0.01019546203315258, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01025, "step": 46, "tokens/total": 1471120, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 20992} +{"epoch": 0.18359375, "grad_norm": 5.682794094085693, "learning_rate": 9.958771393851491e-05, "loss": 0.05226336419582367, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.05365, "step": 47, "tokens/total": 1502272, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 21441} +{"epoch": 0.1875, "grad_norm": 2.1035990715026855, "learning_rate": 9.954758120060702e-05, "loss": 0.03172330930829048, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.03223, "step": 48, "tokens/total": 1534296, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 21901} +{"epoch": 0.19140625, "grad_norm": 0.4331962466239929, "learning_rate": 9.950559465600948e-05, "loss": 0.015544923022389412, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.01567, "step": 49, "tokens/total": 1566544, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 22341} +{"epoch": 0.1953125, "grad_norm": 1.802998423576355, "learning_rate": 9.946175605195379e-05, "loss": 0.02046693116426468, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.02068, "step": 50, "tokens/total": 1601760, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 22833} +{"epoch": 0.19921875, "grad_norm": 0.5978775024414062, "learning_rate": 9.941606721274322e-05, "loss": 0.013067858293652534, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.01315, "step": 51, "tokens/total": 1634312, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 23277} +{"epoch": 0.203125, "grad_norm": 0.9406740069389343, "learning_rate": 9.936853003967685e-05, "loss": 0.015813495963811874, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.01594, "step": 52, "tokens/total": 1668232, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 23759} +{"epoch": 0.20703125, "grad_norm": 4.406692981719971, "learning_rate": 9.93191465109705e-05, "loss": 0.051533982157707214, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.05288, "step": 53, "tokens/total": 1701160, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 24193} +{"epoch": 0.2109375, "grad_norm": 1.062363862991333, "learning_rate": 9.926791868167438e-05, "loss": 0.032096732407808304, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.03262, "step": 54, "tokens/total": 1735968, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 24619} +{"epoch": 0.21484375, "grad_norm": 0.8934375047683716, "learning_rate": 9.921484868358753e-05, "loss": 0.026037417352199554, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.02638, "step": 55, "tokens/total": 1767624, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 25075} +{"epoch": 0.21875, "grad_norm": 0.45804834365844727, "learning_rate": 9.915993872516924e-05, "loss": 0.02011895179748535, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.02032, "step": 56, "tokens/total": 1800536, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 25519} +{"epoch": 0.22265625, "grad_norm": 0.41217613220214844, "learning_rate": 9.9103191091447e-05, "loss": 0.010620267130434513, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01068, "step": 57, "tokens/total": 1830464, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 25966} +{"epoch": 0.2265625, "grad_norm": 3.9141836166381836, "learning_rate": 9.904460814392147e-05, "loss": 0.031039899215102196, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.03153, "step": 58, "tokens/total": 1867568, "tokens/train_per_sec_per_gpu": 7.11, "tokens/trainable": 26423} +{"epoch": 0.23046875, "grad_norm": 0.5704036951065063, "learning_rate": 9.898419232046825e-05, "loss": 0.020633604377508163, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.02085, "step": 59, "tokens/total": 1902424, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 26934} +{"epoch": 0.234375, "grad_norm": 5.1786932945251465, "learning_rate": 9.892194613523633e-05, "loss": 0.018513906747102737, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01869, "step": 60, "tokens/total": 1937032, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 27393} +{"epoch": 0.23828125, "grad_norm": 0.19482901692390442, "learning_rate": 9.885787217854357e-05, "loss": 0.007165948860347271, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00719, "step": 61, "tokens/total": 1968960, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 27852} +{"epoch": 0.2421875, "grad_norm": 1.5671405792236328, "learning_rate": 9.879197311676887e-05, "loss": 0.01801222935318947, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.01818, "step": 62, "tokens/total": 2001688, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 28292} +{"epoch": 0.24609375, "grad_norm": 1.097164511680603, "learning_rate": 9.872425169224113e-05, "loss": 0.023057829588651657, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.02333, "step": 63, "tokens/total": 2034160, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 28798} +{"epoch": 0.25, "grad_norm": 0.4025648534297943, "learning_rate": 9.865471072312528e-05, "loss": 0.016105342656373978, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.01624, "step": 64, "tokens/total": 2069552, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 29283} +{"epoch": 0.25390625, "grad_norm": 0.44939836859703064, "learning_rate": 9.858335310330492e-05, "loss": 0.004237504210323095, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00425, "step": 65, "tokens/total": 2099280, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 29745} +{"epoch": 0.2578125, "grad_norm": 0.5149315595626831, "learning_rate": 9.851018180226185e-05, "loss": 0.00987806636840105, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00993, "step": 66, "tokens/total": 2131280, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 30186} +{"epoch": 0.26171875, "grad_norm": 1.2965457439422607, "learning_rate": 9.843519986495259e-05, "loss": 0.010065598413348198, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.01012, "step": 67, "tokens/total": 2162232, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 30622} +{"epoch": 0.265625, "grad_norm": 0.3872620463371277, "learning_rate": 9.835841041168162e-05, "loss": 0.009195292368531227, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00924, "step": 68, "tokens/total": 2197080, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 31097} +{"epoch": 0.26953125, "grad_norm": 1.217265009880066, "learning_rate": 9.82798166379715e-05, "loss": 0.032946810126304626, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.0335, "step": 69, "tokens/total": 2228208, "tokens/train_per_sec_per_gpu": 10.54, "tokens/trainable": 31595} +{"epoch": 0.2734375, "grad_norm": 0.502037763595581, "learning_rate": 9.819942181443002e-05, "loss": 0.011324050836265087, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.01139, "step": 70, "tokens/total": 2260704, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 32036} +{"epoch": 0.27734375, "grad_norm": 0.4253541827201843, "learning_rate": 9.811722928661392e-05, "loss": 0.009501074440777302, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.00955, "step": 71, "tokens/total": 2288584, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 32428} +{"epoch": 0.28125, "grad_norm": 0.17744335532188416, "learning_rate": 9.803324247488975e-05, "loss": 0.0027247576508671045, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00273, "step": 72, "tokens/total": 2320912, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 32880} +{"epoch": 0.28515625, "grad_norm": 0.11100803315639496, "learning_rate": 9.794746487429161e-05, "loss": 0.0018693532329052687, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.00187, "step": 73, "tokens/total": 2352096, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 33323} +{"epoch": 0.2890625, "grad_norm": 0.3658057749271393, "learning_rate": 9.785990005437554e-05, "loss": 0.009292484261095524, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00934, "step": 74, "tokens/total": 2385328, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 33792} +{"epoch": 0.29296875, "grad_norm": 1.45941960811615, "learning_rate": 9.777055165907117e-05, "loss": 0.020447438582777977, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.02066, "step": 75, "tokens/total": 2416416, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 34223} +{"epoch": 0.296875, "grad_norm": 0.5518165826797485, "learning_rate": 9.767942340652993e-05, "loss": 0.01708814688026905, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01723, "step": 76, "tokens/total": 2446192, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 34649} +{"epoch": 0.30078125, "grad_norm": 0.8281406760215759, "learning_rate": 9.758651908897035e-05, "loss": 0.010761028155684471, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01082, "step": 77, "tokens/total": 2478312, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 35094} +{"epoch": 0.3046875, "grad_norm": 1.4999864101409912, "learning_rate": 9.749184257252033e-05, "loss": 0.048920199275016785, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.05014, "step": 78, "tokens/total": 2509984, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 35534} +{"epoch": 0.30859375, "grad_norm": 1.062252402305603, "learning_rate": 9.739539779705614e-05, "loss": 0.019593777135014534, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.01979, "step": 79, "tokens/total": 2546536, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 36029} +{"epoch": 0.3125, "grad_norm": 0.5459722876548767, "learning_rate": 9.729718877603861e-05, "loss": 0.016822611913084984, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.01696, "step": 80, "tokens/total": 2578048, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 36469} +{"epoch": 0.31640625, "grad_norm": 1.2015717029571533, "learning_rate": 9.719721959634592e-05, "loss": 0.028508855029940605, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.02892, "step": 81, "tokens/total": 2609640, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 36906} +{"epoch": 0.3203125, "grad_norm": 0.40136608481407166, "learning_rate": 9.709549441810375e-05, "loss": 0.007378448732197285, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00741, "step": 82, "tokens/total": 2644408, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 37390} +{"epoch": 0.32421875, "grad_norm": 0.5989400744438171, "learning_rate": 9.699201747451195e-05, "loss": 0.018617138266563416, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01879, "step": 83, "tokens/total": 2675576, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 37838} +{"epoch": 0.328125, "grad_norm": 0.25615739822387695, "learning_rate": 9.688679307166854e-05, "loss": 0.008154037408530712, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00819, "step": 84, "tokens/total": 2706448, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 38310} +{"epoch": 0.33203125, "grad_norm": 1.3001514673233032, "learning_rate": 9.677982558839042e-05, "loss": 0.025581300258636475, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.02591, "step": 85, "tokens/total": 2738824, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 38789} +{"epoch": 0.3359375, "grad_norm": 0.2617473304271698, "learning_rate": 9.66711194760312e-05, "loss": 0.00834340788424015, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00838, "step": 86, "tokens/total": 2775216, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 39277} +{"epoch": 0.33984375, "grad_norm": 0.8706974983215332, "learning_rate": 9.656067925829593e-05, "loss": 0.026894591748714447, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.02726, "step": 87, "tokens/total": 2804720, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 39737} +{"epoch": 0.34375, "grad_norm": 0.5356494784355164, "learning_rate": 9.644850953105288e-05, "loss": 0.010551401413977146, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.01061, "step": 88, "tokens/total": 2833944, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 40179} +{"epoch": 0.34765625, "grad_norm": 0.35498037934303284, "learning_rate": 9.633461496214225e-05, "loss": 0.007831600494682789, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00786, "step": 89, "tokens/total": 2865800, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 40649} +{"epoch": 0.3515625, "grad_norm": 0.4489542543888092, "learning_rate": 9.621900029118195e-05, "loss": 0.021070081740617752, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.02129, "step": 90, "tokens/total": 2894864, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 41080} +{"epoch": 0.35546875, "grad_norm": 27.152555465698242, "learning_rate": 9.610167032937036e-05, "loss": 0.014797612093389034, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.01491, "step": 91, "tokens/total": 2929216, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 41599} +{"epoch": 0.359375, "grad_norm": 0.9511778354644775, "learning_rate": 9.598262995928611e-05, "loss": 0.011865681037306786, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.01194, "step": 92, "tokens/total": 2959280, "tokens/train_per_sec_per_gpu": 10.09, "tokens/trainable": 42076} +{"epoch": 0.36328125, "grad_norm": 1.4305578470230103, "learning_rate": 9.586188413468492e-05, "loss": 0.010653055272996426, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01071, "step": 93, "tokens/total": 2993456, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 42522} +{"epoch": 0.3671875, "grad_norm": 0.3339636027812958, "learning_rate": 9.57394378802934e-05, "loss": 0.00859379954636097, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00863, "step": 94, "tokens/total": 3025752, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 42974} +{"epoch": 0.37109375, "grad_norm": 0.5067156553268433, "learning_rate": 9.56152962916e-05, "loss": 0.025753343477845192, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.02609, "step": 95, "tokens/total": 3054192, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 43380} +{"epoch": 0.375, "grad_norm": 0.2866378128528595, "learning_rate": 9.548946453464296e-05, "loss": 0.006033100187778473, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00605, "step": 96, "tokens/total": 3087160, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 43865} +{"epoch": 0.37890625, "grad_norm": 0.8092994689941406, "learning_rate": 9.53619478457953e-05, "loss": 0.010579278692603111, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01064, "step": 97, "tokens/total": 3121000, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 44328} +{"epoch": 0.3828125, "grad_norm": 0.47828859090805054, "learning_rate": 9.523275153154695e-05, "loss": 0.015792513266205788, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.01592, "step": 98, "tokens/total": 3152272, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 44778} +{"epoch": 0.38671875, "grad_norm": 0.8084412813186646, "learning_rate": 9.51018809682839e-05, "loss": 0.03970591351389885, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.0405, "step": 99, "tokens/total": 3185480, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 45225} +{"epoch": 0.390625, "grad_norm": 0.35319051146507263, "learning_rate": 9.49693416020645e-05, "loss": 0.015568319708108902, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.01569, "step": 100, "tokens/total": 3219360, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 45678} +{"epoch": 0.39453125, "grad_norm": 0.32014504075050354, "learning_rate": 9.483513894839276e-05, "loss": 0.01338218618184328, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01347, "step": 101, "tokens/total": 3251368, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 46125} +{"epoch": 0.3984375, "grad_norm": 0.28175821900367737, "learning_rate": 9.469927859198888e-05, "loss": 0.009314017370343208, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00936, "step": 102, "tokens/total": 3284184, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 46612} +{"epoch": 0.40234375, "grad_norm": 3.2032864093780518, "learning_rate": 9.456176618655689e-05, "loss": 0.017194107174873352, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.01734, "step": 103, "tokens/total": 3314752, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 47059} +{"epoch": 0.40625, "grad_norm": 0.5016529560089111, "learning_rate": 9.442260745454927e-05, "loss": 0.010614384897053242, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.01067, "step": 104, "tokens/total": 3349736, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 47536} +{"epoch": 0.41015625, "grad_norm": 0.6185789704322815, "learning_rate": 9.428180818692884e-05, "loss": 0.014382407069206238, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.01449, "step": 105, "tokens/total": 3384224, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 48037} +{"epoch": 0.4140625, "grad_norm": 1.1049928665161133, "learning_rate": 9.413937424292791e-05, "loss": 0.019440969452261925, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.01963, "step": 106, "tokens/total": 3420808, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 48491} +{"epoch": 0.41796875, "grad_norm": 0.25359758734703064, "learning_rate": 9.399531154980424e-05, "loss": 0.0051986221224069595, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00521, "step": 107, "tokens/total": 3453512, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 48940} +{"epoch": 0.421875, "grad_norm": 0.3563685119152069, "learning_rate": 9.384962610259455e-05, "loss": 0.019742880016565323, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01994, "step": 108, "tokens/total": 3486192, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 49389} +{"epoch": 0.42578125, "grad_norm": 0.09360772371292114, "learning_rate": 9.370232396386494e-05, "loss": 0.0015024960739538074, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.0015, "step": 109, "tokens/total": 3515384, "tokens/train_per_sec_per_gpu": 10.44, "tokens/trainable": 49838} +{"epoch": 0.4296875, "grad_norm": 0.8475779891014099, "learning_rate": 9.355341126345868e-05, "loss": 0.008832360617816448, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00887, "step": 110, "tokens/total": 3548264, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 50310} +{"epoch": 0.43359375, "grad_norm": 0.3574405610561371, "learning_rate": 9.340289419824107e-05, "loss": 0.00784758199006319, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.00788, "step": 111, "tokens/total": 3579288, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 50755} +{"epoch": 0.4375, "grad_norm": 0.5308865308761597, "learning_rate": 9.325077903184159e-05, "loss": 0.015034841373562813, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.01515, "step": 112, "tokens/total": 3611240, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 51213} +{"epoch": 0.44140625, "grad_norm": 0.09035824239253998, "learning_rate": 9.30970720943932e-05, "loss": 0.001638859510421753, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00164, "step": 113, "tokens/total": 3644136, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 51660} +{"epoch": 0.4453125, "grad_norm": 0.43065452575683594, "learning_rate": 9.2941779782269e-05, "loss": 0.004201117437332869, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00421, "step": 114, "tokens/total": 3676368, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 52133} +{"epoch": 0.44921875, "grad_norm": 0.40629854798316956, "learning_rate": 9.278490855781596e-05, "loss": 0.014405193738639355, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.01451, "step": 115, "tokens/total": 3707896, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 52580} +{"epoch": 0.453125, "grad_norm": 1.126939058303833, "learning_rate": 9.262646494908604e-05, "loss": 0.006849400233477354, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00687, "step": 116, "tokens/total": 3742296, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 53033} +{"epoch": 0.45703125, "grad_norm": 0.598143458366394, "learning_rate": 9.246645554956457e-05, "loss": 0.020330185070633888, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.02054, "step": 117, "tokens/total": 3776456, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 53517} +{"epoch": 0.4609375, "grad_norm": 0.30139780044555664, "learning_rate": 9.230488701789578e-05, "loss": 0.002741149626672268, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00274, "step": 118, "tokens/total": 3811688, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 53967} +{"epoch": 0.46484375, "grad_norm": 0.30786821246147156, "learning_rate": 9.214176607760577e-05, "loss": 0.003241901285946369, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00325, "step": 119, "tokens/total": 3847200, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 54430} +{"epoch": 0.46875, "grad_norm": 0.4572126865386963, "learning_rate": 9.197709951682268e-05, "loss": 0.01601463370025158, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.01614, "step": 120, "tokens/total": 3877800, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 54896} +{"epoch": 0.47265625, "grad_norm": 4.063608646392822, "learning_rate": 9.181089418799428e-05, "loss": 0.03697679191827774, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.03767, "step": 121, "tokens/total": 3910976, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 55379} +{"epoch": 0.4765625, "grad_norm": 0.16470958292484283, "learning_rate": 9.164315700760271e-05, "loss": 0.005467007867991924, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00548, "step": 122, "tokens/total": 3943768, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 55803} +{"epoch": 0.48046875, "grad_norm": 0.4766172468662262, "learning_rate": 9.147389495587671e-05, "loss": 0.015342611819505692, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01546, "step": 123, "tokens/total": 3973816, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 56249} +{"epoch": 0.484375, "grad_norm": 0.43708136677742004, "learning_rate": 9.130311507650116e-05, "loss": 0.014974185265600681, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.01509, "step": 124, "tokens/total": 4004704, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 56713} +{"epoch": 0.48828125, "grad_norm": 0.362673282623291, "learning_rate": 9.113082447632394e-05, "loss": 0.007219805382192135, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00725, "step": 125, "tokens/total": 4036040, "tokens/train_per_sec_per_gpu": 10.6, "tokens/trainable": 57196} +{"epoch": 0.4921875, "grad_norm": 1.102900505065918, "learning_rate": 9.09570303250602e-05, "loss": 0.009209919720888138, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00925, "step": 126, "tokens/total": 4066792, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 57680} +{"epoch": 0.49609375, "grad_norm": 0.6331366896629333, "learning_rate": 9.078173985499394e-05, "loss": 0.01167643815279007, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.01174, "step": 127, "tokens/total": 4096112, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 58110} +{"epoch": 0.5, "grad_norm": 0.33725059032440186, "learning_rate": 9.060496036067713e-05, "loss": 0.017414668574929237, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.01757, "step": 128, "tokens/total": 4131472, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 58534} +{"epoch": 0.50390625, "grad_norm": 0.4142087697982788, "learning_rate": 9.042669919862615e-05, "loss": 0.015432886779308319, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.01555, "step": 129, "tokens/total": 4162912, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 58998} +{"epoch": 0.5078125, "grad_norm": 1.006857991218567, "learning_rate": 9.024696378701557e-05, "loss": 0.01680646277964115, "memory/device_reserved (GiB)": 66.81, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.01695, "step": 130, "tokens/total": 4195768, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 59448} +{"epoch": 0.51171875, "grad_norm": 0.2813015580177307, "learning_rate": 9.006576160536948e-05, "loss": 0.006262626964598894, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00628, "step": 131, "tokens/total": 4227192, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 59906} +{"epoch": 0.515625, "grad_norm": 0.3652796745300293, "learning_rate": 8.988310019425035e-05, "loss": 0.010946589522063732, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.01101, "step": 132, "tokens/total": 4263304, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 60350} +{"epoch": 0.51953125, "grad_norm": 0.11750537902116776, "learning_rate": 8.969898715494506e-05, "loss": 0.004046283196657896, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00405, "step": 133, "tokens/total": 4297352, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 60831} +{"epoch": 0.5234375, "grad_norm": 1.0227910280227661, "learning_rate": 8.951343014914869e-05, "loss": 0.008390587754547596, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00843, "step": 134, "tokens/total": 4332152, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 61305} +{"epoch": 0.52734375, "grad_norm": 0.23502375185489655, "learning_rate": 8.932643689864568e-05, "loss": 0.0032589135225862265, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00326, "step": 135, "tokens/total": 4365272, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 61792} +{"epoch": 0.53125, "grad_norm": 0.1045166552066803, "learning_rate": 8.913801518498845e-05, "loss": 0.0010502319782972336, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00105, "step": 136, "tokens/total": 4399928, "tokens/train_per_sec_per_gpu": 6.98, "tokens/trainable": 62253} +{"epoch": 0.53515625, "grad_norm": 0.4404532313346863, "learning_rate": 8.894817284917364e-05, "loss": 0.005304649006575346, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00532, "step": 137, "tokens/total": 4434336, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 62707} +{"epoch": 0.5390625, "grad_norm": 0.27976739406585693, "learning_rate": 8.875691779131569e-05, "loss": 0.010006312280893326, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01006, "step": 138, "tokens/total": 4468872, "tokens/train_per_sec_per_gpu": 6.47, "tokens/trainable": 63145} +{"epoch": 0.54296875, "grad_norm": 0.768483579158783, "learning_rate": 8.856425797031829e-05, "loss": 0.009201083332300186, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00924, "step": 139, "tokens/total": 4502416, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 63587} +{"epoch": 0.546875, "grad_norm": 0.20154857635498047, "learning_rate": 8.837020140354295e-05, "loss": 0.002017534803599119, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00202, "step": 140, "tokens/total": 4536024, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 64052} +{"epoch": 0.55078125, "grad_norm": 1.649791955947876, "learning_rate": 8.817475616647554e-05, "loss": 0.021639764308929443, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.02188, "step": 141, "tokens/total": 4568960, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 64526} +{"epoch": 0.5546875, "grad_norm": 0.5277649760246277, "learning_rate": 8.797793039239017e-05, "loss": 0.015860576182603836, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.01599, "step": 142, "tokens/total": 4601656, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 64983} +{"epoch": 0.55859375, "grad_norm": 0.4205176532268524, "learning_rate": 8.777973227201069e-05, "loss": 0.012677857652306557, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.01276, "step": 143, "tokens/total": 4636136, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 65492} +{"epoch": 0.5625, "grad_norm": 2.3204433917999268, "learning_rate": 8.758017005316988e-05, "loss": 0.009941847063601017, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00999, "step": 144, "tokens/total": 4670160, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 65925} +{"epoch": 0.56640625, "grad_norm": 0.2908189594745636, "learning_rate": 8.737925204046629e-05, "loss": 0.007299849763512611, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.00733, "step": 145, "tokens/total": 4703792, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 66383} +{"epoch": 0.5703125, "grad_norm": 0.27426862716674805, "learning_rate": 8.717698659491851e-05, "loss": 0.006290100049227476, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00631, "step": 146, "tokens/total": 4737640, "tokens/train_per_sec_per_gpu": 7.11, "tokens/trainable": 66840} +{"epoch": 0.57421875, "grad_norm": 3.2931976318359375, "learning_rate": 8.697338213361735e-05, "loss": 0.015411523170769215, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01553, "step": 147, "tokens/total": 4770704, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 67297} +{"epoch": 0.578125, "grad_norm": 2.3439033031463623, "learning_rate": 8.676844712937552e-05, "loss": 0.006160224322229624, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00618, "step": 148, "tokens/total": 4807376, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 67769} +{"epoch": 0.58203125, "grad_norm": 1.4359078407287598, "learning_rate": 8.656219011037509e-05, "loss": 0.025231681764125824, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.02555, "step": 149, "tokens/total": 4840728, "tokens/train_per_sec_per_gpu": 6.69, "tokens/trainable": 68257} +{"epoch": 0.5859375, "grad_norm": 0.2844410240650177, "learning_rate": 8.63546196598125e-05, "loss": 0.0059183030389249325, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00594, "step": 150, "tokens/total": 4873880, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 68706} +{"epoch": 0.58984375, "grad_norm": 0.3426648676395416, "learning_rate": 8.614574441554145e-05, "loss": 0.01161183137446642, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.01168, "step": 151, "tokens/total": 4904512, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 69131} +{"epoch": 0.59375, "grad_norm": 0.27722975611686707, "learning_rate": 8.593557306971349e-05, "loss": 0.005633342079818249, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00565, "step": 152, "tokens/total": 4937048, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 69586} +{"epoch": 0.59765625, "grad_norm": 0.23999489843845367, "learning_rate": 8.572411436841618e-05, "loss": 0.006438427604734898, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00646, "step": 153, "tokens/total": 4968224, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 70061} +{"epoch": 0.6015625, "grad_norm": 0.27180418372154236, "learning_rate": 8.551137711130922e-05, "loss": 0.008264926262199879, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.0083, "step": 154, "tokens/total": 4997272, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 70467} +{"epoch": 0.60546875, "grad_norm": 0.22099076211452484, "learning_rate": 8.529737015125824e-05, "loss": 0.0053521390073001385, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00537, "step": 155, "tokens/total": 5031072, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 70933} +{"epoch": 0.609375, "grad_norm": 0.8613873720169067, "learning_rate": 8.508210239396639e-05, "loss": 0.02627459540963173, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.02662, "step": 156, "tokens/total": 5063808, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 71382} +{"epoch": 0.61328125, "grad_norm": 0.2565155327320099, "learning_rate": 8.486558279760375e-05, "loss": 0.006217642221599817, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00624, "step": 157, "tokens/total": 5094032, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 71808} +{"epoch": 0.6171875, "grad_norm": 0.4332433044910431, "learning_rate": 8.464782037243449e-05, "loss": 0.011610233224928379, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01168, "step": 158, "tokens/total": 5125664, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 72249} +{"epoch": 0.62109375, "grad_norm": 0.4657554030418396, "learning_rate": 8.442882418044202e-05, "loss": 0.011500262655317783, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.01157, "step": 159, "tokens/total": 5156952, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 72726} +{"epoch": 0.625, "grad_norm": 0.29288986325263977, "learning_rate": 8.420860333495179e-05, "loss": 0.008864698000252247, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.0089, "step": 160, "tokens/total": 5187120, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 73148} +{"epoch": 0.62890625, "grad_norm": 0.0767560675740242, "learning_rate": 8.398716700025208e-05, "loss": 0.0011052448535338044, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00111, "step": 161, "tokens/total": 5219080, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 73600} +{"epoch": 0.6328125, "grad_norm": 0.23604854941368103, "learning_rate": 8.376452439121266e-05, "loss": 0.008014293387532234, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.00805, "step": 162, "tokens/total": 5248680, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 74068} +{"epoch": 0.63671875, "grad_norm": 0.17282359302043915, "learning_rate": 8.354068477290124e-05, "loss": 0.004131142050027847, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00414, "step": 163, "tokens/total": 5280104, "tokens/train_per_sec_per_gpu": 10.27, "tokens/trainable": 74527} +{"epoch": 0.640625, "grad_norm": 0.2208026945590973, "learning_rate": 8.331565746019807e-05, "loss": 0.003915427252650261, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00392, "step": 164, "tokens/total": 5311328, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 74992} +{"epoch": 0.64453125, "grad_norm": 0.7049856781959534, "learning_rate": 8.308945181740812e-05, "loss": 0.018065771088004112, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.01823, "step": 165, "tokens/total": 5343960, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 75442} +{"epoch": 0.6484375, "grad_norm": 0.08690542727708817, "learning_rate": 8.286207725787153e-05, "loss": 0.0020215879194438457, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00202, "step": 166, "tokens/total": 5377312, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 75887} +{"epoch": 0.65234375, "grad_norm": 0.1798534393310547, "learning_rate": 8.263354324357182e-05, "loss": 0.003816924523562193, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00382, "step": 167, "tokens/total": 5409824, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 76331} +{"epoch": 0.65625, "grad_norm": 0.36397087574005127, "learning_rate": 8.240385928474219e-05, "loss": 0.010014430619776249, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.01006, "step": 168, "tokens/total": 5440904, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 76745} +{"epoch": 0.66015625, "grad_norm": 0.07774322479963303, "learning_rate": 8.217303493946967e-05, "loss": 0.0006815326632931828, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00068, "step": 169, "tokens/total": 5474920, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 77201} +{"epoch": 0.6640625, "grad_norm": 0.12127338349819183, "learning_rate": 8.194107981329746e-05, "loss": 0.0017105182632803917, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00171, "step": 170, "tokens/total": 5511856, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 77655} +{"epoch": 0.66796875, "grad_norm": 0.0277805645018816, "learning_rate": 8.170800355882518e-05, "loss": 0.0001809945097193122, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00018, "step": 171, "tokens/total": 5544104, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 78122} +{"epoch": 0.671875, "grad_norm": 6.708811283111572, "learning_rate": 8.147381587530713e-05, "loss": 0.027659598737955093, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.02805, "step": 172, "tokens/total": 5577344, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 78576} +{"epoch": 0.67578125, "grad_norm": 0.2788736820220947, "learning_rate": 8.123852650824877e-05, "loss": 0.001918785972520709, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00192, "step": 173, "tokens/total": 5610024, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 79059} +{"epoch": 0.6796875, "grad_norm": 0.8738307356834412, "learning_rate": 8.100214524900103e-05, "loss": 0.018473925068974495, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.01865, "step": 174, "tokens/total": 5643184, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 79498} +{"epoch": 0.68359375, "grad_norm": 0.3490431606769562, "learning_rate": 8.076468193435301e-05, "loss": 0.002075434196740389, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00208, "step": 175, "tokens/total": 5672824, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 79922} +{"epoch": 0.6875, "grad_norm": 0.43081122636795044, "learning_rate": 8.052614644612253e-05, "loss": 0.005601218435913324, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00562, "step": 176, "tokens/total": 5705576, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 80383} +{"epoch": 0.69140625, "grad_norm": 0.40415140986442566, "learning_rate": 8.028654871074489e-05, "loss": 0.0026563075371086597, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00266, "step": 177, "tokens/total": 5737840, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 80824} +{"epoch": 0.6953125, "grad_norm": 0.044670674949884415, "learning_rate": 8.004589869885986e-05, "loss": 0.0004485670360736549, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00045, "step": 178, "tokens/total": 5767552, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 81243} +{"epoch": 0.69921875, "grad_norm": 0.5796677470207214, "learning_rate": 7.980420642489674e-05, "loss": 0.0033357348293066025, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00334, "step": 179, "tokens/total": 5802512, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 81716} +{"epoch": 0.703125, "grad_norm": 0.0035107883159071207, "learning_rate": 7.95614819466576e-05, "loss": 4.370992974145338e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00004, "step": 180, "tokens/total": 5834984, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 82181} +{"epoch": 0.70703125, "grad_norm": 0.13477420806884766, "learning_rate": 7.931773536489872e-05, "loss": 0.0007913983426988125, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00079, "step": 181, "tokens/total": 5867336, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 82626} +{"epoch": 0.7109375, "grad_norm": 0.5716648697853088, "learning_rate": 7.907297682291035e-05, "loss": 0.007407162804156542, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00743, "step": 182, "tokens/total": 5899232, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 83089} +{"epoch": 0.71484375, "grad_norm": 0.4726713001728058, "learning_rate": 7.882721650609442e-05, "loss": 0.012527992948889732, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.01261, "step": 183, "tokens/total": 5934816, "tokens/train_per_sec_per_gpu": 6.79, "tokens/trainable": 83590} +{"epoch": 0.71875, "grad_norm": 1.8613216876983643, "learning_rate": 7.85804646415409e-05, "loss": 0.006289892364293337, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.00631, "step": 184, "tokens/total": 5968120, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 84046} +{"epoch": 0.72265625, "grad_norm": 0.1669372171163559, "learning_rate": 7.833273149760207e-05, "loss": 0.001006398699246347, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00101, "step": 185, "tokens/total": 5998600, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 84517} +{"epoch": 0.7265625, "grad_norm": 0.5793927311897278, "learning_rate": 7.808402738346527e-05, "loss": 0.004986278712749481, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.005, "step": 186, "tokens/total": 6033832, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 84974} +{"epoch": 0.73046875, "grad_norm": 0.027279984205961227, "learning_rate": 7.783436264872382e-05, "loss": 0.00031656777719035745, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00032, "step": 187, "tokens/total": 6068192, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 85460} +{"epoch": 0.734375, "grad_norm": 0.20512546598911285, "learning_rate": 7.758374768294647e-05, "loss": 0.002071539405733347, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00207, "step": 188, "tokens/total": 6101464, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 85939} +{"epoch": 0.73828125, "grad_norm": 0.7749311327934265, "learning_rate": 7.733219291524489e-05, "loss": 0.010034240782260895, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.01008, "step": 189, "tokens/total": 6133008, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 86377} +{"epoch": 0.7421875, "grad_norm": 0.7862029075622559, "learning_rate": 7.707970881383977e-05, "loss": 0.008420512080192566, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.00846, "step": 190, "tokens/total": 6163048, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 86834} +{"epoch": 0.74609375, "grad_norm": 0.7150360345840454, "learning_rate": 7.682630588562518e-05, "loss": 0.01103509683161974, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.0111, "step": 191, "tokens/total": 6196072, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 87265} +{"epoch": 0.75, "grad_norm": 0.9214755892753601, "learning_rate": 7.657199467573129e-05, "loss": 0.01969822682440281, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01989, "step": 192, "tokens/total": 6227312, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 87747} +{"epoch": 0.75390625, "grad_norm": 1.066563606262207, "learning_rate": 7.631678576708561e-05, "loss": 0.01904144324362278, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.01922, "step": 193, "tokens/total": 6260240, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 88239} +{"epoch": 0.7578125, "grad_norm": 0.7288725972175598, "learning_rate": 7.606068977997255e-05, "loss": 0.010701526887714863, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.01076, "step": 194, "tokens/total": 6294280, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 88718} +{"epoch": 0.76171875, "grad_norm": 0.3284401595592499, "learning_rate": 7.580371737159148e-05, "loss": 0.007156226318329573, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00718, "step": 195, "tokens/total": 6323368, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 89129} +{"epoch": 0.765625, "grad_norm": 0.08070968836545944, "learning_rate": 7.554587923561324e-05, "loss": 0.0011974377557635307, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.0012, "step": 196, "tokens/total": 6354960, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 89567} +{"epoch": 0.76953125, "grad_norm": 0.3348805904388428, "learning_rate": 7.528718610173511e-05, "loss": 0.008437501266598701, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.00847, "step": 197, "tokens/total": 6387208, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 89988} +{"epoch": 0.7734375, "grad_norm": 0.3048219084739685, "learning_rate": 7.502764873523431e-05, "loss": 0.002879810519516468, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00288, "step": 198, "tokens/total": 6418720, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 90434} +{"epoch": 0.77734375, "grad_norm": 0.9031469821929932, "learning_rate": 7.476727793652011e-05, "loss": 0.0189344622194767, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.01911, "step": 199, "tokens/total": 6449184, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 90913} +{"epoch": 0.78125, "grad_norm": 0.7931787967681885, "learning_rate": 7.450608454068415e-05, "loss": 0.030804060399532318, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.03128, "step": 200, "tokens/total": 6481824, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 91355} +{"epoch": 0.78515625, "grad_norm": 0.40047791600227356, "learning_rate": 7.424407941704987e-05, "loss": 0.008094128221273422, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00813, "step": 201, "tokens/total": 6515896, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 91856} +{"epoch": 0.7890625, "grad_norm": 0.411347895860672, "learning_rate": 7.398127346871986e-05, "loss": 0.019930489361286163, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.02013, "step": 202, "tokens/total": 6548784, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 92311} +{"epoch": 0.79296875, "grad_norm": 0.9887216091156006, "learning_rate": 7.371767763212238e-05, "loss": 0.003562560770660639, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00357, "step": 203, "tokens/total": 6580256, "tokens/train_per_sec_per_gpu": 10.16, "tokens/trainable": 92764} +{"epoch": 0.796875, "grad_norm": 0.23964789509773254, "learning_rate": 7.345330287655617e-05, "loss": 0.0044450280256569386, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00445, "step": 204, "tokens/total": 6613976, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 93227} +{"epoch": 0.80078125, "grad_norm": 0.6118507385253906, "learning_rate": 7.31881602037339e-05, "loss": 0.018272195011377335, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.01844, "step": 205, "tokens/total": 6644552, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 93675} +{"epoch": 0.8046875, "grad_norm": 0.3382423222064972, "learning_rate": 7.29222606473245e-05, "loss": 0.007393424399197102, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00742, "step": 206, "tokens/total": 6675056, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 94120} +{"epoch": 0.80859375, "grad_norm": 0.43498823046684265, "learning_rate": 7.265561527249383e-05, "loss": 0.015957515686750412, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.01609, "step": 207, "tokens/total": 6706624, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 94557} +{"epoch": 0.8125, "grad_norm": 0.25760090351104736, "learning_rate": 7.238823517544436e-05, "loss": 0.007530502043664455, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00756, "step": 208, "tokens/total": 6741008, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 95035} +{"epoch": 0.81640625, "grad_norm": 0.138915553689003, "learning_rate": 7.212013148295333e-05, "loss": 0.0031206500716507435, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00313, "step": 209, "tokens/total": 6774872, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 95517} +{"epoch": 0.8203125, "grad_norm": 1.9327337741851807, "learning_rate": 7.185131535190975e-05, "loss": 0.006946972105652094, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00697, "step": 210, "tokens/total": 6805816, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 96026} +{"epoch": 0.82421875, "grad_norm": 0.3322914242744446, "learning_rate": 7.158179796885005e-05, "loss": 0.004068898502737284, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00408, "step": 211, "tokens/total": 6837592, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 96477} +{"epoch": 0.828125, "grad_norm": 0.04263783246278763, "learning_rate": 7.131159054949273e-05, "loss": 0.000513539940584451, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00051, "step": 212, "tokens/total": 6874384, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 96951} +{"epoch": 0.83203125, "grad_norm": 0.09751681983470917, "learning_rate": 7.104070433827139e-05, "loss": 0.0015660293865948915, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00157, "step": 213, "tokens/total": 6905872, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 97350} +{"epoch": 0.8359375, "grad_norm": 0.08044496178627014, "learning_rate": 7.076915060786705e-05, "loss": 0.00164657027926296, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.00165, "step": 214, "tokens/total": 6937984, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 97792} +{"epoch": 0.83984375, "grad_norm": 0.24836711585521698, "learning_rate": 7.049694065873882e-05, "loss": 0.002967417472973466, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00297, "step": 215, "tokens/total": 6970704, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 98257} +{"epoch": 0.84375, "grad_norm": 0.7372098565101624, "learning_rate": 7.022408581865382e-05, "loss": 0.019062677398324013, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.01925, "step": 216, "tokens/total": 7004616, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 98731} +{"epoch": 0.84765625, "grad_norm": 0.08871692419052124, "learning_rate": 6.99505974422157e-05, "loss": 0.001554459915496409, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00156, "step": 217, "tokens/total": 7036880, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 99212} +{"epoch": 0.8515625, "grad_norm": 0.11803542077541351, "learning_rate": 6.967648691039213e-05, "loss": 0.0013171113096177578, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00132, "step": 218, "tokens/total": 7068208, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 99654} +{"epoch": 0.85546875, "grad_norm": 0.025526180863380432, "learning_rate": 6.940176563004123e-05, "loss": 0.0004314134712330997, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00043, "step": 219, "tokens/total": 7098112, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 100086} +{"epoch": 0.859375, "grad_norm": 0.13033103942871094, "learning_rate": 6.912644503343682e-05, "loss": 0.001251000794582069, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00125, "step": 220, "tokens/total": 7132432, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 100524} +{"epoch": 0.86328125, "grad_norm": 0.1456366777420044, "learning_rate": 6.885053657779273e-05, "loss": 0.001508563756942749, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.00151, "step": 221, "tokens/total": 7164016, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 100996} +{"epoch": 0.8671875, "grad_norm": 0.05492303892970085, "learning_rate": 6.857405174478604e-05, "loss": 0.0007669074111618102, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00077, "step": 222, "tokens/total": 7198344, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 101449} +{"epoch": 0.87109375, "grad_norm": 0.3415512144565582, "learning_rate": 6.82970020400792e-05, "loss": 0.010739377699792385, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.0108, "step": 223, "tokens/total": 7232192, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 101905} +{"epoch": 0.875, "grad_norm": 0.05250921845436096, "learning_rate": 6.801939899284132e-05, "loss": 0.0005125325405970216, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00051, "step": 224, "tokens/total": 7266984, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 102411} +{"epoch": 0.87890625, "grad_norm": 0.020981961861252785, "learning_rate": 6.774125415526827e-05, "loss": 0.0003328109160065651, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00033, "step": 225, "tokens/total": 7299512, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 102882} +{"epoch": 0.8828125, "grad_norm": 0.27784475684165955, "learning_rate": 6.746257910210214e-05, "loss": 0.0017177170375362039, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00172, "step": 226, "tokens/total": 7331120, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 103332} +{"epoch": 0.88671875, "grad_norm": 0.03268355503678322, "learning_rate": 6.718338543014937e-05, "loss": 0.0005118112894706428, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00051, "step": 227, "tokens/total": 7360504, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 103790} +{"epoch": 0.890625, "grad_norm": 1.201480746269226, "learning_rate": 6.69036847577983e-05, "loss": 0.005038812756538391, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00505, "step": 228, "tokens/total": 7390760, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 104246} +{"epoch": 0.89453125, "grad_norm": 0.3806019127368927, "learning_rate": 6.662348872453553e-05, "loss": 0.007126105949282646, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00715, "step": 229, "tokens/total": 7423352, "tokens/train_per_sec_per_gpu": 10.6, "tokens/trainable": 104707} +{"epoch": 0.8984375, "grad_norm": 0.01582666113972664, "learning_rate": 6.63428089904618e-05, "loss": 0.00020338728791102767, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.0002, "step": 230, "tokens/total": 7457152, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 105167} +{"epoch": 0.90234375, "grad_norm": 0.023667845875024796, "learning_rate": 6.60616572358065e-05, "loss": 0.0004092699382454157, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00041, "step": 231, "tokens/total": 7488648, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 105576} +{"epoch": 0.90625, "grad_norm": 0.08793454617261887, "learning_rate": 6.578004516044172e-05, "loss": 0.0007972054299898446, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.0008, "step": 232, "tokens/total": 7522160, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 106043} +{"epoch": 0.91015625, "grad_norm": 0.2669622004032135, "learning_rate": 6.549798448339548e-05, "loss": 0.004440234042704105, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00445, "step": 233, "tokens/total": 7557336, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 106506} +{"epoch": 0.9140625, "grad_norm": 0.42330074310302734, "learning_rate": 6.521548694236384e-05, "loss": 0.004364547785371542, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00437, "step": 234, "tokens/total": 7590432, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 106951} +{"epoch": 0.91796875, "grad_norm": 0.496096134185791, "learning_rate": 6.493256429322259e-05, "loss": 0.005625514779239893, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00564, "step": 235, "tokens/total": 7623240, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 107382} +{"epoch": 0.921875, "grad_norm": 0.2180124670267105, "learning_rate": 6.464922830953799e-05, "loss": 0.002950491849333048, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00295, "step": 236, "tokens/total": 7657968, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 107814} +{"epoch": 0.92578125, "grad_norm": 0.5746164917945862, "learning_rate": 6.436549078207688e-05, "loss": 0.027319906279444695, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.0277, "step": 237, "tokens/total": 7689320, "tokens/train_per_sec_per_gpu": 10.22, "tokens/trainable": 108274} +{"epoch": 0.9296875, "grad_norm": 0.4922912120819092, "learning_rate": 6.408136351831592e-05, "loss": 0.008090472780168056, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00812, "step": 238, "tokens/total": 7721568, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 108714} +{"epoch": 0.93359375, "grad_norm": 0.1255158632993698, "learning_rate": 6.379685834195036e-05, "loss": 0.0006707434658892453, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00067, "step": 239, "tokens/total": 7756464, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 109220} +{"epoch": 0.9375, "grad_norm": 0.15411542356014252, "learning_rate": 6.351198709240186e-05, "loss": 0.0021162498742341995, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00212, "step": 240, "tokens/total": 7791616, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 109672} +{"epoch": 0.94140625, "grad_norm": 0.10442740470170975, "learning_rate": 6.32267616243259e-05, "loss": 0.0016446950612589717, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00165, "step": 241, "tokens/total": 7823520, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 110135} +{"epoch": 0.9453125, "grad_norm": 0.17225496470928192, "learning_rate": 6.294119380711849e-05, "loss": 0.001354136737063527, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00136, "step": 242, "tokens/total": 7854424, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 110563} +{"epoch": 0.94921875, "grad_norm": 0.81834876537323, "learning_rate": 6.265529552442209e-05, "loss": 0.03017202392220497, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.03063, "step": 243, "tokens/total": 7887160, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 111049} +{"epoch": 0.953125, "grad_norm": 0.13129891455173492, "learning_rate": 6.236907867363127e-05, "loss": 0.0011790544958785176, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00118, "step": 244, "tokens/total": 7918736, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 111495} +{"epoch": 0.95703125, "grad_norm": 0.751765251159668, "learning_rate": 6.208255516539749e-05, "loss": 0.0018926660995930433, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00189, "step": 245, "tokens/total": 7949448, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 111968} +{"epoch": 0.9609375, "grad_norm": 0.12610264122486115, "learning_rate": 6.179573692313344e-05, "loss": 0.001700294786132872, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.0017, "step": 246, "tokens/total": 7984680, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 112416} +{"epoch": 0.96484375, "grad_norm": 0.06522932648658752, "learning_rate": 6.150863588251694e-05, "loss": 0.001169363735243678, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00117, "step": 247, "tokens/total": 8019816, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 112882} +{"epoch": 0.96875, "grad_norm": 15.50298023223877, "learning_rate": 6.122126399099419e-05, "loss": 0.002635641023516655, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00264, "step": 248, "tokens/total": 8056272, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 113390} +{"epoch": 0.97265625, "grad_norm": 0.48287588357925415, "learning_rate": 6.0933633207282615e-05, "loss": 0.012180381454527378, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.01225, "step": 249, "tokens/total": 8090224, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 113904} +{"epoch": 0.9765625, "grad_norm": 0.10979297757148743, "learning_rate": 6.064575550087316e-05, "loss": 0.0021856448147445917, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00219, "step": 250, "tokens/total": 8124528, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 114351} +{"epoch": 0.98046875, "grad_norm": 0.101508729159832, "learning_rate": 6.0357642851532245e-05, "loss": 0.003198768710717559, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.0032, "step": 251, "tokens/total": 8159688, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 114842} +{"epoch": 0.984375, "grad_norm": 0.2556680142879486, "learning_rate": 6.0069307248803294e-05, "loss": 0.005299929995089769, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00531, "step": 252, "tokens/total": 8191800, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 115263} +{"epoch": 0.98828125, "grad_norm": 0.09119684994220734, "learning_rate": 5.9780760691507635e-05, "loss": 0.0031947866082191467, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.0032, "step": 253, "tokens/total": 8221752, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 115703} +{"epoch": 0.9921875, "grad_norm": 0.10158612579107285, "learning_rate": 5.9492015187245334e-05, "loss": 0.0019558051135390997, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00196, "step": 254, "tokens/total": 8255648, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 116157} +{"epoch": 0.99609375, "grad_norm": 0.18203122913837433, "learning_rate": 5.920308275189541e-05, "loss": 0.00279613328166306, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.0028, "step": 255, "tokens/total": 8286664, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 116567} +{"epoch": 1.0, "grad_norm": 0.023021891713142395, "learning_rate": 5.8913975409115874e-05, "loss": 0.00046330521581694484, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00046, "step": 256, "tokens/total": 8318368, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 117030} +{"epoch": 1.00390625, "grad_norm": 0.03635823354125023, "learning_rate": 5.8624705189843395e-05, "loss": 0.0005449475138448179, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00055, "step": 257, "tokens/total": 8353336, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 117517} +{"epoch": 1.0078125, "grad_norm": 0.1097673699259758, "learning_rate": 5.833528413179249e-05, "loss": 0.0006327102310024202, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00063, "step": 258, "tokens/total": 8389112, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 118015} +{"epoch": 1.01171875, "grad_norm": 0.03970041498541832, "learning_rate": 5.80457242789548e-05, "loss": 0.0003919436130672693, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00039, "step": 259, "tokens/total": 8426000, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 118520} +{"epoch": 1.015625, "grad_norm": 0.05850733816623688, "learning_rate": 5.77560376810977e-05, "loss": 0.0005852528847754002, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.00059, "step": 260, "tokens/total": 8458568, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 118992} +{"epoch": 1.01953125, "grad_norm": 0.004649393726140261, "learning_rate": 5.7466236393263005e-05, "loss": 8.723568316781893e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00009, "step": 261, "tokens/total": 8490840, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 119438} +{"epoch": 1.0234375, "grad_norm": 0.013800840824842453, "learning_rate": 5.717633247526522e-05, "loss": 0.0001782537146937102, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00018, "step": 262, "tokens/total": 8521792, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 119881} +{"epoch": 1.02734375, "grad_norm": 0.00471842335537076, "learning_rate": 5.688633799118971e-05, "loss": 8.161615551216528e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00008, "step": 263, "tokens/total": 8551336, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 120285} +{"epoch": 1.03125, "grad_norm": 0.03571309894323349, "learning_rate": 5.659626500889066e-05, "loss": 0.00021831589401699603, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00022, "step": 264, "tokens/total": 8579872, "tokens/train_per_sec_per_gpu": 7.09, "tokens/trainable": 120755} +{"epoch": 1.03515625, "grad_norm": 0.0028788791969418526, "learning_rate": 5.6306125599488905e-05, "loss": 3.477300197118893e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00003, "step": 265, "tokens/total": 8608920, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 121194} +{"epoch": 1.0390625, "grad_norm": 0.29093337059020996, "learning_rate": 5.601593183686955e-05, "loss": 0.0012748365988954902, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00128, "step": 266, "tokens/total": 8643600, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 121670} +{"epoch": 1.04296875, "grad_norm": 0.030671032145619392, "learning_rate": 5.572569579717961e-05, "loss": 0.0002542288857512176, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00025, "step": 267, "tokens/total": 8676032, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 122142} +{"epoch": 1.046875, "grad_norm": 0.0024570876266807318, "learning_rate": 5.543542955832538e-05, "loss": 2.4141081667039543e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00002, "step": 268, "tokens/total": 8713240, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 122585} +{"epoch": 1.05078125, "grad_norm": 1.2305799722671509, "learning_rate": 5.514514519946986e-05, "loss": 0.00306285684928298, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00307, "step": 269, "tokens/total": 8751664, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 123091} +{"epoch": 1.0546875, "grad_norm": 0.0656183660030365, "learning_rate": 5.485485480053015e-05, "loss": 0.00028290535556152463, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00028, "step": 270, "tokens/total": 8786112, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 123505} +{"epoch": 1.05859375, "grad_norm": 0.004044094122946262, "learning_rate": 5.4564570441674645e-05, "loss": 2.061039958789479e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00002, "step": 271, "tokens/total": 8816920, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 123953} +{"epoch": 1.0625, "grad_norm": 0.0016182539984583855, "learning_rate": 5.42743042028204e-05, "loss": 2.0705263523268513e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00002, "step": 272, "tokens/total": 8851408, "tokens/train_per_sec_per_gpu": 6.86, "tokens/trainable": 124400} +{"epoch": 1.06640625, "grad_norm": 0.15029816329479218, "learning_rate": 5.3984068163130464e-05, "loss": 0.0009096303838305175, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00091, "step": 273, "tokens/total": 8883768, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 124870} +{"epoch": 1.0703125, "grad_norm": 0.0015225159004330635, "learning_rate": 5.369387440051111e-05, "loss": 1.6997273633023724e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00002, "step": 274, "tokens/total": 8916200, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 125333} +{"epoch": 1.07421875, "grad_norm": 0.0014344003284350038, "learning_rate": 5.340373499110935e-05, "loss": 1.3540368854592089e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00001, "step": 275, "tokens/total": 8949920, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 125834} +{"epoch": 1.078125, "grad_norm": 0.000812255428172648, "learning_rate": 5.3113662008810304e-05, "loss": 5.317499926604796e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00001, "step": 276, "tokens/total": 8986448, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 126316} +{"epoch": 1.08203125, "grad_norm": 0.03341461718082428, "learning_rate": 5.282366752473479e-05, "loss": 0.0001283441815758124, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00013, "step": 277, "tokens/total": 9020600, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 126798} +{"epoch": 1.0859375, "grad_norm": 0.006137323100119829, "learning_rate": 5.2533763606737005e-05, "loss": 2.287784627696965e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00002, "step": 278, "tokens/total": 9056040, "tokens/train_per_sec_per_gpu": 6.92, "tokens/trainable": 127223} +{"epoch": 1.08984375, "grad_norm": 0.0351930670440197, "learning_rate": 5.224396231890232e-05, "loss": 0.00016301203868351877, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00016, "step": 279, "tokens/total": 9087072, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 127672} +{"epoch": 1.09375, "grad_norm": 0.16618295013904572, "learning_rate": 5.195427572104522e-05, "loss": 0.00048031078767962754, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00048, "step": 280, "tokens/total": 9118376, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 128116} +{"epoch": 1.09765625, "grad_norm": 0.8451645374298096, "learning_rate": 5.166471586820751e-05, "loss": 0.004286065697669983, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.0043, "step": 281, "tokens/total": 9152704, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 128589} +{"epoch": 1.1015625, "grad_norm": 2.1212587356567383, "learning_rate": 5.1375294810156615e-05, "loss": 0.01706903800368309, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.01722, "step": 282, "tokens/total": 9184448, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 129036} +{"epoch": 1.10546875, "grad_norm": 0.008543205447494984, "learning_rate": 5.1086024590884144e-05, "loss": 5.101683927932754e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00005, "step": 283, "tokens/total": 9217392, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 129485} +{"epoch": 1.109375, "grad_norm": 0.7681220173835754, "learning_rate": 5.079691724810461e-05, "loss": 0.009021918289363384, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00906, "step": 284, "tokens/total": 9252016, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 129920} +{"epoch": 1.11328125, "grad_norm": 0.008588019758462906, "learning_rate": 5.0507984812754684e-05, "loss": 3.868219209834933e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00004, "step": 285, "tokens/total": 9284008, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 130417} +{"epoch": 1.1171875, "grad_norm": 0.05393043905496597, "learning_rate": 5.021923930849237e-05, "loss": 0.000292272656224668, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00029, "step": 286, "tokens/total": 9315984, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 130903} +{"epoch": 1.12109375, "grad_norm": 0.017236333340406418, "learning_rate": 4.99306927511967e-05, "loss": 7.587523577967659e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00008, "step": 287, "tokens/total": 9347376, "tokens/train_per_sec_per_gpu": 10.5, "tokens/trainable": 131343} +{"epoch": 1.125, "grad_norm": 0.024503551423549652, "learning_rate": 4.964235714846775e-05, "loss": 0.00010152401955565438, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.0001, "step": 288, "tokens/total": 9377424, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 131763} +{"epoch": 1.12890625, "grad_norm": 0.1296500563621521, "learning_rate": 4.9354244499126866e-05, "loss": 0.0008079413091763854, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00081, "step": 289, "tokens/total": 9408104, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 132229} +{"epoch": 1.1328125, "grad_norm": 0.0014188096392899752, "learning_rate": 4.90663667927174e-05, "loss": 1.698547202977352e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00002, "step": 290, "tokens/total": 9439944, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 132678} +{"epoch": 1.13671875, "grad_norm": 0.2607637941837311, "learning_rate": 4.877873600900581e-05, "loss": 0.0006435546674765646, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00064, "step": 291, "tokens/total": 9472104, "tokens/train_per_sec_per_gpu": 6.27, "tokens/trainable": 133136} +{"epoch": 1.140625, "grad_norm": 0.01244624424725771, "learning_rate": 4.849136411748306e-05, "loss": 4.126537896809168e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00004, "step": 292, "tokens/total": 9508336, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 133608} +{"epoch": 1.14453125, "grad_norm": 0.003969596233218908, "learning_rate": 4.8204263076866574e-05, "loss": 2.7101988962385803e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00003, "step": 293, "tokens/total": 9541552, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 134108} +{"epoch": 1.1484375, "grad_norm": 0.20917832851409912, "learning_rate": 4.791744483460251e-05, "loss": 0.001150335418060422, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00115, "step": 294, "tokens/total": 9575072, "tokens/train_per_sec_per_gpu": 6.63, "tokens/trainable": 134541} +{"epoch": 1.15234375, "grad_norm": 0.012095500715076923, "learning_rate": 4.7630921326368736e-05, "loss": 4.8389381845481694e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00005, "step": 295, "tokens/total": 9610344, "tokens/train_per_sec_per_gpu": 6.47, "tokens/trainable": 134966} +{"epoch": 1.15625, "grad_norm": 0.26763588190078735, "learning_rate": 4.7344704475577916e-05, "loss": 0.00230082543566823, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.0023, "step": 296, "tokens/total": 9639728, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 135402} +{"epoch": 1.16015625, "grad_norm": 0.5118128657341003, "learning_rate": 4.705880619288153e-05, "loss": 0.005054690409451723, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00507, "step": 297, "tokens/total": 9671424, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 135804} +{"epoch": 1.1640625, "grad_norm": 0.13468582928180695, "learning_rate": 4.677323837567412e-05, "loss": 0.0010927437106147408, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00109, "step": 298, "tokens/total": 9700808, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 136231} +{"epoch": 1.16796875, "grad_norm": 0.16426074504852295, "learning_rate": 4.6488012907598146e-05, "loss": 0.0012849201448261738, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00129, "step": 299, "tokens/total": 9732312, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 136705} +{"epoch": 1.171875, "grad_norm": 1.1424593925476074, "learning_rate": 4.620314165804964e-05, "loss": 0.01218369696289301, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01226, "step": 300, "tokens/total": 9764624, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 137178} +{"epoch": 1.17578125, "grad_norm": 0.0676557719707489, "learning_rate": 4.591863648168407e-05, "loss": 0.0003689370641950518, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00037, "step": 301, "tokens/total": 9798368, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 137643} +{"epoch": 1.1796875, "grad_norm": 0.0021677068434655666, "learning_rate": 4.5634509217923135e-05, "loss": 2.919624421338085e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00003, "step": 302, "tokens/total": 9830192, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 138078} +{"epoch": 1.18359375, "grad_norm": 0.1398150771856308, "learning_rate": 4.535077169046201e-05, "loss": 0.0014520471449941397, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00145, "step": 303, "tokens/total": 9863112, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 138515} +{"epoch": 1.1875, "grad_norm": 0.730445921421051, "learning_rate": 4.506743570677743e-05, "loss": 0.015471857972443104, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.01559, "step": 304, "tokens/total": 9893192, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 138948} +{"epoch": 1.19140625, "grad_norm": 0.31661081314086914, "learning_rate": 4.478451305763618e-05, "loss": 0.0017309405375272036, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00173, "step": 305, "tokens/total": 9925712, "tokens/train_per_sec_per_gpu": 9.88, "tokens/trainable": 139408} +{"epoch": 1.1953125, "grad_norm": 0.005134343635290861, "learning_rate": 4.450201551660454e-05, "loss": 7.651090709259734e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00008, "step": 306, "tokens/total": 9955736, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 139840} +{"epoch": 1.19921875, "grad_norm": 0.24792981147766113, "learning_rate": 4.4219954839558276e-05, "loss": 0.0015690438449382782, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00157, "step": 307, "tokens/total": 9987160, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 140281} +{"epoch": 1.203125, "grad_norm": 0.007184525951743126, "learning_rate": 4.393834276419352e-05, "loss": 4.4246371544431895e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00004, "step": 308, "tokens/total": 10017360, "tokens/train_per_sec_per_gpu": 11.21, "tokens/trainable": 140776} +{"epoch": 1.20703125, "grad_norm": 0.008083555847406387, "learning_rate": 4.36571910095382e-05, "loss": 5.9045029047410935e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00006, "step": 309, "tokens/total": 10052056, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 141228} +{"epoch": 1.2109375, "grad_norm": 0.3572210669517517, "learning_rate": 4.337651127546448e-05, "loss": 0.0015788939781486988, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00158, "step": 310, "tokens/total": 10085424, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 141679} +{"epoch": 1.21484375, "grad_norm": 0.07022370398044586, "learning_rate": 4.3096315242201736e-05, "loss": 0.000271849799901247, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00027, "step": 311, "tokens/total": 10116824, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 142122} +{"epoch": 1.21875, "grad_norm": 0.004137261770665646, "learning_rate": 4.2816614569850635e-05, "loss": 3.517814911901951e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00004, "step": 312, "tokens/total": 10150376, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 142561} +{"epoch": 1.22265625, "grad_norm": 0.8779999017715454, "learning_rate": 4.2537420897897864e-05, "loss": 0.03451008349657059, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.03511, "step": 313, "tokens/total": 10184432, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 143046} +{"epoch": 1.2265625, "grad_norm": 0.1394595354795456, "learning_rate": 4.225874584473174e-05, "loss": 0.0013762768357992172, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00138, "step": 314, "tokens/total": 10214848, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 143493} +{"epoch": 1.23046875, "grad_norm": 0.015281321480870247, "learning_rate": 4.19806010071587e-05, "loss": 9.309763845521957e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00009, "step": 315, "tokens/total": 10247952, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 143956} +{"epoch": 1.234375, "grad_norm": 0.27160122990608215, "learning_rate": 4.170299795992081e-05, "loss": 0.0015040471917018294, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00151, "step": 316, "tokens/total": 10280648, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 144411} +{"epoch": 1.23828125, "grad_norm": 0.2776739299297333, "learning_rate": 4.142594825521398e-05, "loss": 0.003255086950957775, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00326, "step": 317, "tokens/total": 10317712, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 144892} +{"epoch": 1.2421875, "grad_norm": 0.06863569468259811, "learning_rate": 4.114946342220728e-05, "loss": 0.0006230767467059195, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00062, "step": 318, "tokens/total": 10351632, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 145339} +{"epoch": 1.24609375, "grad_norm": 0.11241894960403442, "learning_rate": 4.087355496656321e-05, "loss": 0.0008129080524668097, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00081, "step": 319, "tokens/total": 10383336, "tokens/train_per_sec_per_gpu": 10.25, "tokens/trainable": 145811} +{"epoch": 1.25, "grad_norm": 0.42912429571151733, "learning_rate": 4.05982343699588e-05, "loss": 0.006244337186217308, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00626, "step": 320, "tokens/total": 10418016, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 146314} +{"epoch": 1.25390625, "grad_norm": 0.09895195066928864, "learning_rate": 4.0323513089607876e-05, "loss": 0.000451402273029089, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00045, "step": 321, "tokens/total": 10451600, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 146781} +{"epoch": 1.2578125, "grad_norm": 0.11048132181167603, "learning_rate": 4.004940255778431e-05, "loss": 0.001686611445620656, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00169, "step": 322, "tokens/total": 10487568, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 147225} +{"epoch": 1.26171875, "grad_norm": 0.06768201291561127, "learning_rate": 3.977591418134619e-05, "loss": 0.0008173336973413825, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00082, "step": 323, "tokens/total": 10520976, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 147673} +{"epoch": 1.265625, "grad_norm": 0.014912494458258152, "learning_rate": 3.95030593412612e-05, "loss": 0.00025589336291886866, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00026, "step": 324, "tokens/total": 10552776, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 148103} +{"epoch": 1.26953125, "grad_norm": 0.3571690618991852, "learning_rate": 3.923084939213296e-05, "loss": 0.004809688311070204, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00482, "step": 325, "tokens/total": 10582576, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 148535} +{"epoch": 1.2734375, "grad_norm": 0.04223630577325821, "learning_rate": 3.895929566172861e-05, "loss": 0.0006324055138975382, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00063, "step": 326, "tokens/total": 10615600, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 148999} +{"epoch": 1.27734375, "grad_norm": 0.028869694098830223, "learning_rate": 3.868840945050728e-05, "loss": 0.0005100031266920269, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00051, "step": 327, "tokens/total": 10650688, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 149431} +{"epoch": 1.28125, "grad_norm": 0.20389939844608307, "learning_rate": 3.841820203114995e-05, "loss": 0.0006023422465659678, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.0006, "step": 328, "tokens/total": 10681816, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 149886} +{"epoch": 1.28515625, "grad_norm": 0.2543167769908905, "learning_rate": 3.814868464809027e-05, "loss": 0.004407048691064119, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00442, "step": 329, "tokens/total": 10711728, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 150288} +{"epoch": 1.2890625, "grad_norm": 0.15902657806873322, "learning_rate": 3.787986851704667e-05, "loss": 0.0025331603828817606, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00254, "step": 330, "tokens/total": 10744952, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 150733} +{"epoch": 1.29296875, "grad_norm": 0.012967195361852646, "learning_rate": 3.7611764824555654e-05, "loss": 0.0001273458037758246, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00013, "step": 331, "tokens/total": 10776192, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 151207} +{"epoch": 1.296875, "grad_norm": 0.07004442065954208, "learning_rate": 3.734438472750619e-05, "loss": 0.0002781406801659614, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00028, "step": 332, "tokens/total": 10813336, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 151694} +{"epoch": 1.30078125, "grad_norm": 0.016194703057408333, "learning_rate": 3.707773935267552e-05, "loss": 0.00020421543740667403, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.0002, "step": 333, "tokens/total": 10847808, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 152188} +{"epoch": 1.3046875, "grad_norm": 0.007617066614329815, "learning_rate": 3.68118397962661e-05, "loss": 6.322658009594306e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00006, "step": 334, "tokens/total": 10879896, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 152596} +{"epoch": 1.30859375, "grad_norm": 0.1854279339313507, "learning_rate": 3.654669712344384e-05, "loss": 0.0010690134949982166, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00107, "step": 335, "tokens/total": 10908976, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 153052} +{"epoch": 1.3125, "grad_norm": 0.06483833491802216, "learning_rate": 3.628232236787763e-05, "loss": 0.0006728494772687554, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00067, "step": 336, "tokens/total": 10943152, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 153491} +{"epoch": 1.31640625, "grad_norm": 0.021753253415226936, "learning_rate": 3.6018726531280144e-05, "loss": 0.0001802305632736534, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00018, "step": 337, "tokens/total": 10978184, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 153983} +{"epoch": 1.3203125, "grad_norm": 0.02377374656498432, "learning_rate": 3.575592058295017e-05, "loss": 0.00016909789701458067, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00017, "step": 338, "tokens/total": 11010056, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 154464} +{"epoch": 1.32421875, "grad_norm": 0.0035628939513117075, "learning_rate": 3.549391545931585e-05, "loss": 3.937931978725828e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00004, "step": 339, "tokens/total": 11040968, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 154924} +{"epoch": 1.328125, "grad_norm": 0.025660144165158272, "learning_rate": 3.5232722063479914e-05, "loss": 0.00014723747153766453, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00015, "step": 340, "tokens/total": 11073424, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 155373} +{"epoch": 1.33203125, "grad_norm": 0.06795816123485565, "learning_rate": 3.49723512647657e-05, "loss": 0.0006440329598262906, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00064, "step": 341, "tokens/total": 11109224, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 155873} +{"epoch": 1.3359375, "grad_norm": 0.004735002759844065, "learning_rate": 3.471281389826491e-05, "loss": 6.298009975580499e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00006, "step": 342, "tokens/total": 11139856, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 156278} +{"epoch": 1.33984375, "grad_norm": 0.009534417651593685, "learning_rate": 3.4454120764386764e-05, "loss": 0.00016935849271249026, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00017, "step": 343, "tokens/total": 11175224, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 156733} +{"epoch": 1.34375, "grad_norm": 0.008065089583396912, "learning_rate": 3.4196282628408526e-05, "loss": 9.020716970553622e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00009, "step": 344, "tokens/total": 11207112, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 157203} +{"epoch": 1.34765625, "grad_norm": 0.09096349775791168, "learning_rate": 3.3939310220027456e-05, "loss": 0.0005259146564640105, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00053, "step": 345, "tokens/total": 11240432, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 157707} +{"epoch": 1.3515625, "grad_norm": 0.044807128608226776, "learning_rate": 3.3683214232914404e-05, "loss": 4.324400288169272e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00004, "step": 346, "tokens/total": 11274584, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 158180} +{"epoch": 1.35546875, "grad_norm": 0.024082589894533157, "learning_rate": 3.342800532426873e-05, "loss": 0.00015241288929246366, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00015, "step": 347, "tokens/total": 11307376, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 158646} +{"epoch": 1.359375, "grad_norm": 0.489565908908844, "learning_rate": 3.317369411437484e-05, "loss": 0.0008632151875644922, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00086, "step": 348, "tokens/total": 11340640, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 159115} +{"epoch": 1.36328125, "grad_norm": 0.03917158395051956, "learning_rate": 3.292029118616024e-05, "loss": 0.00017675066192168742, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00018, "step": 349, "tokens/total": 11371632, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 159538} +{"epoch": 1.3671875, "grad_norm": 0.02417062781751156, "learning_rate": 3.266780708475511e-05, "loss": 0.00011468225420685485, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00011, "step": 350, "tokens/total": 11404248, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 160016} +{"epoch": 1.37109375, "grad_norm": 0.006015353370457888, "learning_rate": 3.241625231705354e-05, "loss": 8.168013300746679e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00008, "step": 351, "tokens/total": 11439552, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 160475} +{"epoch": 1.375, "grad_norm": 0.048681821674108505, "learning_rate": 3.216563735127618e-05, "loss": 0.00033827280276454985, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00034, "step": 352, "tokens/total": 11470944, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 160952} +{"epoch": 1.37890625, "grad_norm": 0.5314064621925354, "learning_rate": 3.191597261653475e-05, "loss": 0.009950371459126472, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01, "step": 353, "tokens/total": 11506304, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 161420} +{"epoch": 1.3828125, "grad_norm": 0.010795005597174168, "learning_rate": 3.166726850239794e-05, "loss": 0.00010915935854427516, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00011, "step": 354, "tokens/total": 11535424, "tokens/train_per_sec_per_gpu": 11.09, "tokens/trainable": 161868} +{"epoch": 1.38671875, "grad_norm": 0.010858340188860893, "learning_rate": 3.141953535845912e-05, "loss": 3.722190740518272e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00004, "step": 355, "tokens/total": 11565176, "tokens/train_per_sec_per_gpu": 10.34, "tokens/trainable": 162318} +{"epoch": 1.390625, "grad_norm": 0.01259853970259428, "learning_rate": 3.11727834939056e-05, "loss": 0.00016184768173843622, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00016, "step": 356, "tokens/total": 11595528, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 162789} +{"epoch": 1.39453125, "grad_norm": 0.31451815366744995, "learning_rate": 3.092702317708967e-05, "loss": 0.014799922704696655, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.01491, "step": 357, "tokens/total": 11627248, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 163254} +{"epoch": 1.3984375, "grad_norm": 0.09751182794570923, "learning_rate": 3.0682264635101276e-05, "loss": 0.0006779341492801905, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00068, "step": 358, "tokens/total": 11659440, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 163715} +{"epoch": 1.40234375, "grad_norm": 0.30571845173835754, "learning_rate": 3.0438518053342407e-05, "loss": 0.0005275081493891776, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00053, "step": 359, "tokens/total": 11693288, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 164197} +{"epoch": 1.40625, "grad_norm": 0.01420314610004425, "learning_rate": 3.0195793575103266e-05, "loss": 0.00023237447021529078, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00023, "step": 360, "tokens/total": 11730888, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 164661} +{"epoch": 1.41015625, "grad_norm": 0.16624805331230164, "learning_rate": 2.9954101301140146e-05, "loss": 0.0008943856810219586, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00089, "step": 361, "tokens/total": 11765656, "tokens/train_per_sec_per_gpu": 6.97, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.005288227461278439, "learning_rate": 2.9713451289255123e-05, "loss": 8.828882710076869e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00009, "step": 362, "tokens/total": 11793456, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 165552} +{"epoch": 1.41796875, "grad_norm": 0.03537282720208168, "learning_rate": 2.9473853553877484e-05, "loss": 0.0003736349754035473, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00037, "step": 363, "tokens/total": 11826280, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 166026} +{"epoch": 1.421875, "grad_norm": 0.061419934034347534, "learning_rate": 2.9235318065647e-05, "loss": 0.00051350292051211, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00051, "step": 364, "tokens/total": 11863552, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 166486} +{"epoch": 1.42578125, "grad_norm": 0.006926677189767361, "learning_rate": 2.8997854750998964e-05, "loss": 4.353742770035751e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00004, "step": 365, "tokens/total": 11898616, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 166959} +{"epoch": 1.4296875, "grad_norm": 0.021322835236787796, "learning_rate": 2.8761473491751258e-05, "loss": 8.491883636452258e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00008, "step": 366, "tokens/total": 11930728, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 167394} +{"epoch": 1.43359375, "grad_norm": 0.32576727867126465, "learning_rate": 2.8526184124692883e-05, "loss": 0.0010835954453796148, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00108, "step": 367, "tokens/total": 11964976, "tokens/train_per_sec_per_gpu": 6.45, "tokens/trainable": 167832} +{"epoch": 1.4375, "grad_norm": 0.009217794984579086, "learning_rate": 2.829199644117484e-05, "loss": 8.47414048621431e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00008, "step": 368, "tokens/total": 11996696, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 168242} +{"epoch": 1.44140625, "grad_norm": 0.01105561014264822, "learning_rate": 2.8058920186702553e-05, "loss": 9.278460493078455e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00009, "step": 369, "tokens/total": 12033432, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 168715} +{"epoch": 1.4453125, "grad_norm": 0.03543733432888985, "learning_rate": 2.782696506053033e-05, "loss": 0.00021132681285962462, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00021, "step": 370, "tokens/total": 12067616, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 169147} +{"epoch": 1.44921875, "grad_norm": 0.1202218234539032, "learning_rate": 2.7596140715257824e-05, "loss": 0.0013496584724634886, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00135, "step": 371, "tokens/total": 12099008, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 169628} +{"epoch": 1.453125, "grad_norm": 0.003082921029999852, "learning_rate": 2.7366456756428184e-05, "loss": 4.7086956328712404e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00005, "step": 372, "tokens/total": 12131304, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 170085} +{"epoch": 1.45703125, "grad_norm": 0.010793342255055904, "learning_rate": 2.7137922742128486e-05, "loss": 0.00010829712846316397, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00011, "step": 373, "tokens/total": 12164104, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 170584} +{"epoch": 1.4609375, "grad_norm": 0.44240015745162964, "learning_rate": 2.691054818259188e-05, "loss": 0.002133750356733799, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00214, "step": 374, "tokens/total": 12197776, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 171058} +{"epoch": 1.46484375, "grad_norm": 0.13552218675613403, "learning_rate": 2.6684342539801933e-05, "loss": 0.0004190168692730367, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00042, "step": 375, "tokens/total": 12228064, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 171518} +{"epoch": 1.46875, "grad_norm": 0.01669352315366268, "learning_rate": 2.645931522709877e-05, "loss": 0.0001062872470356524, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00011, "step": 376, "tokens/total": 12261688, "tokens/train_per_sec_per_gpu": 10.81, "tokens/trainable": 172007} +{"epoch": 1.47265625, "grad_norm": 0.05798530578613281, "learning_rate": 2.6235475608787365e-05, "loss": 0.00014682055916637182, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00015, "step": 377, "tokens/total": 12294240, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 172468} +{"epoch": 1.4765625, "grad_norm": 0.11452044546604156, "learning_rate": 2.6012832999747916e-05, "loss": 0.0008997636032290757, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.0009, "step": 378, "tokens/total": 12324528, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 172923} +{"epoch": 1.48046875, "grad_norm": 0.4565379321575165, "learning_rate": 2.579139666504821e-05, "loss": 0.01184794306755066, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01192, "step": 379, "tokens/total": 12360288, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 173370} +{"epoch": 1.484375, "grad_norm": 0.037760816514492035, "learning_rate": 2.557117581955798e-05, "loss": 0.00020454540208447725, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.0002, "step": 380, "tokens/total": 12391216, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 173837} +{"epoch": 1.48828125, "grad_norm": 0.009703868068754673, "learning_rate": 2.5352179627565532e-05, "loss": 5.998317647026852e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00006, "step": 381, "tokens/total": 12423624, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 174294} +{"epoch": 1.4921875, "grad_norm": 0.0022936721798032522, "learning_rate": 2.5134417202396277e-05, "loss": 3.014307549165096e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00003, "step": 382, "tokens/total": 12456032, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 174701} +{"epoch": 1.49609375, "grad_norm": 0.8699129819869995, "learning_rate": 2.491789760603361e-05, "loss": 0.00019177401554770768, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00019, "step": 383, "tokens/total": 12489336, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 175131} +{"epoch": 1.5, "grad_norm": 0.07474718987941742, "learning_rate": 2.4702629848741764e-05, "loss": 0.0006451529916375875, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00065, "step": 384, "tokens/total": 12521944, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 175586} +{"epoch": 1.50390625, "grad_norm": 1.2499576807022095, "learning_rate": 2.4488622888690785e-05, "loss": 0.004298178944736719, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00431, "step": 385, "tokens/total": 12554024, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 176026} +{"epoch": 1.5078125, "grad_norm": 0.006110694725066423, "learning_rate": 2.427588563158384e-05, "loss": 7.513562013627961e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00008, "step": 386, "tokens/total": 12587104, "tokens/train_per_sec_per_gpu": 7.24, "tokens/trainable": 176512} +{"epoch": 1.51171875, "grad_norm": 0.13167335093021393, "learning_rate": 2.406442693028651e-05, "loss": 0.0017258836887776852, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00173, "step": 387, "tokens/total": 12617456, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 176943} +{"epoch": 1.515625, "grad_norm": 0.0019490973791107535, "learning_rate": 2.3854255584458547e-05, "loss": 3.828628541668877e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00004, "step": 388, "tokens/total": 12652696, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 177370} +{"epoch": 1.51953125, "grad_norm": 0.008307907730340958, "learning_rate": 2.3645380340187508e-05, "loss": 0.0001229142799274996, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00012, "step": 389, "tokens/total": 12685792, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 177851} +{"epoch": 1.5234375, "grad_norm": 0.3265420198440552, "learning_rate": 2.3437809889624914e-05, "loss": 0.00425843195989728, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00427, "step": 390, "tokens/total": 12716120, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 178310} +{"epoch": 1.52734375, "grad_norm": 0.23863168060779572, "learning_rate": 2.3231552870624487e-05, "loss": 0.0012714244658127427, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00127, "step": 391, "tokens/total": 12746216, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 178736} +{"epoch": 1.53125, "grad_norm": 0.3201853930950165, "learning_rate": 2.3026617866382657e-05, "loss": 0.007909595966339111, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00794, "step": 392, "tokens/total": 12778592, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 179233} +{"epoch": 1.53515625, "grad_norm": 0.1373405009508133, "learning_rate": 2.2823013405081507e-05, "loss": 0.0008188091451302171, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00082, "step": 393, "tokens/total": 12812008, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 179730} +{"epoch": 1.5390625, "grad_norm": 0.014858311042189598, "learning_rate": 2.2620747959533722e-05, "loss": 0.00010100042709382251, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.0001, "step": 394, "tokens/total": 12844264, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 180227} +{"epoch": 1.54296875, "grad_norm": 0.002070705173537135, "learning_rate": 2.2419829946830123e-05, "loss": 5.202340253163129e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00005, "step": 395, "tokens/total": 12878328, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 180687} +{"epoch": 1.546875, "grad_norm": 0.09738792479038239, "learning_rate": 2.2220267727989325e-05, "loss": 0.0008164338651113212, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00082, "step": 396, "tokens/total": 12907912, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 181141} +{"epoch": 1.55078125, "grad_norm": 0.056369420140981674, "learning_rate": 2.202206960760984e-05, "loss": 0.0006840850110165775, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00068, "step": 397, "tokens/total": 12941968, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 181648} +{"epoch": 1.5546875, "grad_norm": 0.01733791083097458, "learning_rate": 2.182524383352446e-05, "loss": 0.00018290584557689726, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00018, "step": 398, "tokens/total": 12975240, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 182109} +{"epoch": 1.55859375, "grad_norm": 0.023470435291528702, "learning_rate": 2.1629798596457056e-05, "loss": 0.00028449861565604806, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00028, "step": 399, "tokens/total": 13006912, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 182559} +{"epoch": 1.5625, "grad_norm": 0.003011935856193304, "learning_rate": 2.1435742029681725e-05, "loss": 4.588937372318469e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00005, "step": 400, "tokens/total": 13037640, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 182985} +{"epoch": 1.56640625, "grad_norm": 0.002946007763966918, "learning_rate": 2.124308220868431e-05, "loss": 4.708559936261736e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00005, "step": 401, "tokens/total": 13070608, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 183440} +{"epoch": 1.5703125, "grad_norm": 0.021986978128552437, "learning_rate": 2.105182715082638e-05, "loss": 0.000331881019519642, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00033, "step": 402, "tokens/total": 13101784, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 183917} +{"epoch": 1.57421875, "grad_norm": 0.019258596003055573, "learning_rate": 2.0861984815011552e-05, "loss": 0.00021801164257340133, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00022, "step": 403, "tokens/total": 13133528, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 184348} +{"epoch": 1.578125, "grad_norm": 0.0048769996501505375, "learning_rate": 2.0673563101354323e-05, "loss": 9.786394366528839e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.0001, "step": 404, "tokens/total": 13167088, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 184786} +{"epoch": 1.58203125, "grad_norm": 0.006737562827765942, "learning_rate": 2.0486569850851317e-05, "loss": 8.418704965151846e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00008, "step": 405, "tokens/total": 13200392, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 185249} +{"epoch": 1.5859375, "grad_norm": 0.09579817205667496, "learning_rate": 2.0301012845054956e-05, "loss": 0.0010108562419191003, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00101, "step": 406, "tokens/total": 13235488, "tokens/train_per_sec_per_gpu": 6.92, "tokens/trainable": 185680} +{"epoch": 1.58984375, "grad_norm": 0.008364113047719002, "learning_rate": 2.011689980574966e-05, "loss": 9.124513599090278e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00009, "step": 407, "tokens/total": 13269416, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 186100} +{"epoch": 1.59375, "grad_norm": 0.04555624723434448, "learning_rate": 1.993423839463052e-05, "loss": 0.00039926727185957134, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.0004, "step": 408, "tokens/total": 13301360, "tokens/train_per_sec_per_gpu": 10.51, "tokens/trainable": 186565} +{"epoch": 1.59765625, "grad_norm": 0.010548657737672329, "learning_rate": 1.975303621298445e-05, "loss": 8.880461973603815e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00009, "step": 409, "tokens/total": 13336392, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 187017} +{"epoch": 1.6015625, "grad_norm": 0.004035205580294132, "learning_rate": 1.957330080137385e-05, "loss": 6.086752182454802e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00006, "step": 410, "tokens/total": 13366072, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 187468} +{"epoch": 1.60546875, "grad_norm": 0.018689610064029694, "learning_rate": 1.9395039639322864e-05, "loss": 0.00021353308693505824, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00021, "step": 411, "tokens/total": 13400832, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 187961} +{"epoch": 1.609375, "grad_norm": 0.0018486619228497148, "learning_rate": 1.9218260145006073e-05, "loss": 3.5215234674979e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00004, "step": 412, "tokens/total": 13434920, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 188447} +{"epoch": 1.61328125, "grad_norm": 0.007779833395034075, "learning_rate": 1.904296967493982e-05, "loss": 6.574126746272668e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00007, "step": 413, "tokens/total": 13466512, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 188903} +{"epoch": 1.6171875, "grad_norm": 0.008685566484928131, "learning_rate": 1.8869175523676064e-05, "loss": 8.900561078917235e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00009, "step": 414, "tokens/total": 13500352, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 189398} +{"epoch": 1.62109375, "grad_norm": 0.012351326644420624, "learning_rate": 1.869688492349885e-05, "loss": 0.00017204444156959653, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00017, "step": 415, "tokens/total": 13532448, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 189850} +{"epoch": 1.625, "grad_norm": 0.37306952476501465, "learning_rate": 1.85261050441233e-05, "loss": 0.005321073811501265, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00534, "step": 416, "tokens/total": 13565648, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 190306} +{"epoch": 1.62890625, "grad_norm": 0.026526520028710365, "learning_rate": 1.8356842992397304e-05, "loss": 0.0002473396889399737, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00025, "step": 417, "tokens/total": 13597600, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 190746} +{"epoch": 1.6328125, "grad_norm": 0.01687876507639885, "learning_rate": 1.8189105812005714e-05, "loss": 0.00010628512245602906, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00011, "step": 418, "tokens/total": 13630456, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 191156} +{"epoch": 1.63671875, "grad_norm": 0.003959721885621548, "learning_rate": 1.802290048317732e-05, "loss": 5.133161903358996e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00005, "step": 419, "tokens/total": 13660896, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 191573} +{"epoch": 1.640625, "grad_norm": 0.10373859107494354, "learning_rate": 1.785823392239424e-05, "loss": 0.000353733281372115, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00035, "step": 420, "tokens/total": 13693176, "tokens/train_per_sec_per_gpu": 10.04, "tokens/trainable": 192014} +{"epoch": 1.64453125, "grad_norm": 0.2238238900899887, "learning_rate": 1.7695112982104225e-05, "loss": 0.0033529011998325586, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00336, "step": 421, "tokens/total": 13725400, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 192453} +{"epoch": 1.6484375, "grad_norm": 0.023552240803837776, "learning_rate": 1.7533544450435433e-05, "loss": 8.383407111978158e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00008, "step": 422, "tokens/total": 13752752, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 192886} +{"epoch": 1.65234375, "grad_norm": 0.0012259274953976274, "learning_rate": 1.7373535050913946e-05, "loss": 1.4441255189012736e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00001, "step": 423, "tokens/total": 13786136, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 193323} +{"epoch": 1.65625, "grad_norm": 0.1526416391134262, "learning_rate": 1.721509144218405e-05, "loss": 0.0008331008139066398, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00083, "step": 424, "tokens/total": 13815904, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 193753} +{"epoch": 1.66015625, "grad_norm": 0.02506130374968052, "learning_rate": 1.705822021773101e-05, "loss": 0.0003588383842725307, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00036, "step": 425, "tokens/total": 13847976, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 194240} +{"epoch": 1.6640625, "grad_norm": 0.10819127410650253, "learning_rate": 1.69029279056068e-05, "loss": 0.0008251433609984815, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00083, "step": 426, "tokens/total": 13883200, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 194680} +{"epoch": 1.66796875, "grad_norm": 0.008914267644286156, "learning_rate": 1.6749220968158415e-05, "loss": 4.268707562005147e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00004, "step": 427, "tokens/total": 13916080, "tokens/train_per_sec_per_gpu": 10.45, "tokens/trainable": 195186} +{"epoch": 1.671875, "grad_norm": 0.016633326187729836, "learning_rate": 1.659710580175893e-05, "loss": 0.00013722613221034408, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00014, "step": 428, "tokens/total": 13946448, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 195693} +{"epoch": 1.67578125, "grad_norm": 0.03819442167878151, "learning_rate": 1.644658873654133e-05, "loss": 0.00023308282834477723, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00023, "step": 429, "tokens/total": 13975696, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 196130} +{"epoch": 1.6796875, "grad_norm": 0.0061171031557023525, "learning_rate": 1.629767603613508e-05, "loss": 5.9968631830997765e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00006, "step": 430, "tokens/total": 14002616, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 196530} +{"epoch": 1.68359375, "grad_norm": 0.01172108668833971, "learning_rate": 1.615037389740547e-05, "loss": 0.00012155032891314477, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00012, "step": 431, "tokens/total": 14034848, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 196974} +{"epoch": 1.6875, "grad_norm": 0.04699543118476868, "learning_rate": 1.600468845019576e-05, "loss": 0.0004927447880618274, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00049, "step": 432, "tokens/total": 14069032, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 197454} +{"epoch": 1.69140625, "grad_norm": 0.0009191330755129457, "learning_rate": 1.5860625757072092e-05, "loss": 1.4972065400797874e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00001, "step": 433, "tokens/total": 14101296, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 197902} +{"epoch": 1.6953125, "grad_norm": 0.0006092582480050623, "learning_rate": 1.571819181307116e-05, "loss": 1.2628532203962095e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00001, "step": 434, "tokens/total": 14135880, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 198374} +{"epoch": 1.69921875, "grad_norm": 0.007944373413920403, "learning_rate": 1.557739254545075e-05, "loss": 9.623473306419328e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.0001, "step": 435, "tokens/total": 14166040, "tokens/train_per_sec_per_gpu": 10.71, "tokens/trainable": 198846} +{"epoch": 1.703125, "grad_norm": 0.0509788803756237, "learning_rate": 1.543823381344311e-05, "loss": 0.00029470387380570173, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00029, "step": 436, "tokens/total": 14196032, "tokens/train_per_sec_per_gpu": 10.84, "tokens/trainable": 199320} +{"epoch": 1.70703125, "grad_norm": 0.053430844098329544, "learning_rate": 1.5300721408011114e-05, "loss": 0.000493441300932318, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00049, "step": 437, "tokens/total": 14228008, "tokens/train_per_sec_per_gpu": 10.29, "tokens/trainable": 199777} +{"epoch": 1.7109375, "grad_norm": 0.0012507423525676131, "learning_rate": 1.5164861051607254e-05, "loss": 2.0336039597168565e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.76, "memory/max_allocated (GiB)": 65.76, "ppl": 1.00002, "step": 438, "tokens/total": 14261744, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 200194} +{"epoch": 1.71484375, "grad_norm": 0.0054288748651742935, "learning_rate": 1.5030658397935521e-05, "loss": 3.463370376266539e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00003, "step": 439, "tokens/total": 14295208, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 200663} +{"epoch": 1.71875, "grad_norm": 0.0016928918194025755, "learning_rate": 1.4898119031716104e-05, "loss": 2.832262725860346e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00003, "step": 440, "tokens/total": 14327264, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 201121} +{"epoch": 1.72265625, "grad_norm": 0.0651768147945404, "learning_rate": 1.476724846845306e-05, "loss": 0.0003730087773874402, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00037, "step": 441, "tokens/total": 14363088, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 201598} +{"epoch": 1.7265625, "grad_norm": 0.002993773203343153, "learning_rate": 1.463805215420471e-05, "loss": 4.8717978643253446e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00005, "step": 442, "tokens/total": 14396088, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 202068} +{"epoch": 1.73046875, "grad_norm": 0.006570516619831324, "learning_rate": 1.451053546535705e-05, "loss": 3.575946175260469e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00004, "step": 443, "tokens/total": 14429232, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 202524} +{"epoch": 1.734375, "grad_norm": 0.009060447104275227, "learning_rate": 1.438470370840001e-05, "loss": 6.716260395478457e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00007, "step": 444, "tokens/total": 14463264, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 202996} +{"epoch": 1.73828125, "grad_norm": 0.027208028361201286, "learning_rate": 1.4260562119706606e-05, "loss": 0.00018903988529928029, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00019, "step": 445, "tokens/total": 14493552, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 203470} +{"epoch": 1.7421875, "grad_norm": 0.10143569111824036, "learning_rate": 1.413811586531508e-05, "loss": 0.0006037225248292089, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.0006, "step": 446, "tokens/total": 14525240, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 203919} +{"epoch": 1.74609375, "grad_norm": 0.3994045555591583, "learning_rate": 1.4017370040713884e-05, "loss": 0.004443727433681488, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00445, "step": 447, "tokens/total": 14559560, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 204376} +{"epoch": 1.75, "grad_norm": 0.014023041352629662, "learning_rate": 1.3898329670629645e-05, "loss": 5.418311047833413e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00005, "step": 448, "tokens/total": 14594160, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 204821} +{"epoch": 1.75390625, "grad_norm": 0.0009638054762035608, "learning_rate": 1.3780999708818058e-05, "loss": 1.238720869878307e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00001, "step": 449, "tokens/total": 14624224, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 205271} +{"epoch": 1.7578125, "grad_norm": 0.0015416833339259028, "learning_rate": 1.3665385037857758e-05, "loss": 1.9262988644186407e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00002, "step": 450, "tokens/total": 14654984, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 205723} +{"epoch": 1.76171875, "grad_norm": 0.023376356810331345, "learning_rate": 1.3551490468947126e-05, "loss": 0.0002068118192255497, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00021, "step": 451, "tokens/total": 14685440, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 206169} +{"epoch": 1.765625, "grad_norm": 0.01394282840192318, "learning_rate": 1.3439320741704075e-05, "loss": 7.968740101205185e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00008, "step": 452, "tokens/total": 14716120, "tokens/train_per_sec_per_gpu": 10.01, "tokens/trainable": 206592} +{"epoch": 1.76953125, "grad_norm": 0.002701675985008478, "learning_rate": 1.3328880523968808e-05, "loss": 3.072375329793431e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00003, "step": 453, "tokens/total": 14748424, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 207055} +{"epoch": 1.7734375, "grad_norm": 0.05774581432342529, "learning_rate": 1.3220174411609587e-05, "loss": 7.689742051297799e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00008, "step": 454, "tokens/total": 14781256, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 207494} +{"epoch": 1.77734375, "grad_norm": 0.003067963058128953, "learning_rate": 1.3113206928331471e-05, "loss": 3.0446572054643184e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00003, "step": 455, "tokens/total": 14809552, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 207956} +{"epoch": 1.78125, "grad_norm": 0.003858761163428426, "learning_rate": 1.300798252548806e-05, "loss": 2.486477023921907e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00002, "step": 456, "tokens/total": 14841808, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 208408} +{"epoch": 1.78515625, "grad_norm": 0.0035047954879701138, "learning_rate": 1.2904505581896265e-05, "loss": 2.6362369681010023e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00003, "step": 457, "tokens/total": 14875896, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 208889} +{"epoch": 1.7890625, "grad_norm": 0.007627851329743862, "learning_rate": 1.2802780403654082e-05, "loss": 0.00010552673484198749, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00011, "step": 458, "tokens/total": 14908424, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 209336} +{"epoch": 1.79296875, "grad_norm": 0.03310441970825195, "learning_rate": 1.2702811223961408e-05, "loss": 8.065060683293268e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00008, "step": 459, "tokens/total": 14939168, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 209797} +{"epoch": 1.796875, "grad_norm": 0.030235128477215767, "learning_rate": 1.2604602202943861e-05, "loss": 0.00018094989354722202, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00018, "step": 460, "tokens/total": 14969872, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 210240} +{"epoch": 1.80078125, "grad_norm": 0.0020503669511526823, "learning_rate": 1.2508157427479686e-05, "loss": 2.6649564460967667e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00003, "step": 461, "tokens/total": 15001472, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 210702} +{"epoch": 1.8046875, "grad_norm": 0.0576530322432518, "learning_rate": 1.2413480911029655e-05, "loss": 0.0005178657011128962, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00052, "step": 462, "tokens/total": 15030592, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 211140} +{"epoch": 1.80859375, "grad_norm": 0.03816575929522514, "learning_rate": 1.2320576593470082e-05, "loss": 0.00036593739059753716, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00037, "step": 463, "tokens/total": 15063648, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 211575} +{"epoch": 1.8125, "grad_norm": 0.0009832630166783929, "learning_rate": 1.2229448340928828e-05, "loss": 8.722919119463768e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00001, "step": 464, "tokens/total": 15098248, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 212083} +{"epoch": 1.81640625, "grad_norm": 0.0015575995203107595, "learning_rate": 1.2140099945624458e-05, "loss": 2.1833189748576842e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00002, "step": 465, "tokens/total": 15133216, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 212558} +{"epoch": 1.8203125, "grad_norm": 0.0013978495262563229, "learning_rate": 1.205253512570841e-05, "loss": 2.7176265575690195e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00003, "step": 466, "tokens/total": 15165416, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 213014} +{"epoch": 1.82421875, "grad_norm": 0.018338026478886604, "learning_rate": 1.1966757525110255e-05, "loss": 0.00013784610200673342, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00014, "step": 467, "tokens/total": 15193688, "tokens/train_per_sec_per_gpu": 10.18, "tokens/trainable": 213465} +{"epoch": 1.828125, "grad_norm": 0.16325406730175018, "learning_rate": 1.1882770713386095e-05, "loss": 0.002062337240204215, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00206, "step": 468, "tokens/total": 15228600, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 213940} +{"epoch": 1.83203125, "grad_norm": 0.008876061998307705, "learning_rate": 1.180057818556998e-05, "loss": 5.086033706902526e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00005, "step": 469, "tokens/total": 15262192, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 214415} +{"epoch": 1.8359375, "grad_norm": 0.013819404877722263, "learning_rate": 1.1720183362028494e-05, "loss": 0.00010209860192844644, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.0001, "step": 470, "tokens/total": 15294920, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 214880} +{"epoch": 1.83984375, "grad_norm": 0.054750196635723114, "learning_rate": 1.1641589588318387e-05, "loss": 0.00043812196236103773, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00044, "step": 471, "tokens/total": 15327432, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 215334} +{"epoch": 1.84375, "grad_norm": 0.04153619334101677, "learning_rate": 1.1564800135047418e-05, "loss": 0.000126065278891474, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00013, "step": 472, "tokens/total": 15359928, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 215822} +{"epoch": 1.84765625, "grad_norm": 0.3831973075866699, "learning_rate": 1.148981819773816e-05, "loss": 0.0018711215816438198, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00187, "step": 473, "tokens/total": 15391400, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 216239} +{"epoch": 1.8515625, "grad_norm": 0.019581008702516556, "learning_rate": 1.1416646896695086e-05, "loss": 0.00013020502228755504, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00013, "step": 474, "tokens/total": 15424144, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 216691} +{"epoch": 1.85546875, "grad_norm": 0.004207144491374493, "learning_rate": 1.1345289276874717e-05, "loss": 6.607246905332431e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00007, "step": 475, "tokens/total": 15457680, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 217162} +{"epoch": 1.859375, "grad_norm": 0.030784539878368378, "learning_rate": 1.1275748307758873e-05, "loss": 0.000108051419374533, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00011, "step": 476, "tokens/total": 15492056, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 217638} +{"epoch": 1.86328125, "grad_norm": 0.003004812402650714, "learning_rate": 1.1208026883231147e-05, "loss": 2.068038156721741e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00002, "step": 477, "tokens/total": 15521544, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 218053} +{"epoch": 1.8671875, "grad_norm": 0.0501369945704937, "learning_rate": 1.1142127821456433e-05, "loss": 0.00035180593840777874, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00035, "step": 478, "tokens/total": 15553320, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 218503} +{"epoch": 1.87109375, "grad_norm": 0.010655323974788189, "learning_rate": 1.1078053864763674e-05, "loss": 5.487902308232151e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00005, "step": 479, "tokens/total": 15586208, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 218964} +{"epoch": 1.875, "grad_norm": 0.04429743066430092, "learning_rate": 1.1015807679531756e-05, "loss": 0.00018412050849292427, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00018, "step": 480, "tokens/total": 15616432, "tokens/train_per_sec_per_gpu": 10.05, "tokens/trainable": 219412} +{"epoch": 1.87890625, "grad_norm": 0.6061050295829773, "learning_rate": 1.0955391856078528e-05, "loss": 0.006567395757883787, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00659, "step": 481, "tokens/total": 15650056, "tokens/train_per_sec_per_gpu": 10.37, "tokens/trainable": 219886} +{"epoch": 1.8828125, "grad_norm": 0.01046677678823471, "learning_rate": 1.0896808908553007e-05, "loss": 4.4705851905746385e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00004, "step": 482, "tokens/total": 15680336, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 220343} +{"epoch": 1.88671875, "grad_norm": 0.01999562792479992, "learning_rate": 1.0840061274830763e-05, "loss": 4.736140908789821e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00005, "step": 483, "tokens/total": 15711552, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 220825} +{"epoch": 1.890625, "grad_norm": 0.0010406390065327287, "learning_rate": 1.0785151316412473e-05, "loss": 8.864475603331812e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00001, "step": 484, "tokens/total": 15744664, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 221282} +{"epoch": 1.89453125, "grad_norm": 0.15306644141674042, "learning_rate": 1.0732081318325639e-05, "loss": 0.0007786605856381357, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00078, "step": 485, "tokens/total": 15782800, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 221780} +{"epoch": 1.8984375, "grad_norm": 0.005576255731284618, "learning_rate": 1.0680853489029501e-05, "loss": 5.2955194405512884e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00005, "step": 486, "tokens/total": 15817368, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 222210} +{"epoch": 1.90234375, "grad_norm": 0.00410723639652133, "learning_rate": 1.0631469960323152e-05, "loss": 4.098063436686061e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00004, "step": 487, "tokens/total": 15850408, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 222646} +{"epoch": 1.90625, "grad_norm": 0.3574463427066803, "learning_rate": 1.0583932787256783e-05, "loss": 0.0037499882746487856, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00376, "step": 488, "tokens/total": 15881448, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 223066} +{"epoch": 1.91015625, "grad_norm": 0.0030592745169997215, "learning_rate": 1.0538243948046206e-05, "loss": 2.4267432308988646e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00002, "step": 489, "tokens/total": 15913736, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 223512} +{"epoch": 1.9140625, "grad_norm": 0.0009805456502363086, "learning_rate": 1.0494405343990523e-05, "loss": 1.5221250578179024e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00002, "step": 490, "tokens/total": 15944568, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 223956} +{"epoch": 1.91796875, "grad_norm": 0.011790803633630276, "learning_rate": 1.0452418799392985e-05, "loss": 7.58066526032053e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00008, "step": 491, "tokens/total": 15976352, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 224416} +{"epoch": 1.921875, "grad_norm": 0.07875607162714005, "learning_rate": 1.0412286061485102e-05, "loss": 0.00033223192440345883, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00033, "step": 492, "tokens/total": 16010392, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 224883} +{"epoch": 1.92578125, "grad_norm": 0.002662611659616232, "learning_rate": 1.03740088003539e-05, "loss": 1.8680711946217343e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00002, "step": 493, "tokens/total": 16039176, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 225324} +{"epoch": 1.9296875, "grad_norm": 0.000666041683871299, "learning_rate": 1.0337588608872463e-05, "loss": 1.2159572179371025e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00001, "step": 494, "tokens/total": 16067928, "tokens/train_per_sec_per_gpu": 10.61, "tokens/trainable": 225783} +{"epoch": 1.93359375, "grad_norm": 0.4403429329395294, "learning_rate": 1.0303027002633622e-05, "loss": 0.0033286195248365402, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00333, "step": 495, "tokens/total": 16100264, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 226281} +{"epoch": 1.9375, "grad_norm": 0.0005165223847143352, "learning_rate": 1.0270325419886884e-05, "loss": 1.165052890428342e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00001, "step": 496, "tokens/total": 16132504, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 226735} +{"epoch": 1.94140625, "grad_norm": 0.010141369886696339, "learning_rate": 1.0239485221478599e-05, "loss": 8.143248851411045e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00008, "step": 497, "tokens/total": 16167064, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 227198} +{"epoch": 1.9453125, "grad_norm": 0.0019222188275307417, "learning_rate": 1.0210507690795292e-05, "loss": 1.6901274648262188e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00002, "step": 498, "tokens/total": 16198584, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 227623} +{"epoch": 1.94921875, "grad_norm": 0.017252815887331963, "learning_rate": 1.0183394033710305e-05, "loss": 0.00012565456563606858, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00013, "step": 499, "tokens/total": 16229584, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 228074} +{"epoch": 1.953125, "grad_norm": 0.21313342452049255, "learning_rate": 1.0158145378533583e-05, "loss": 0.0024836966767907143, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00249, "step": 500, "tokens/total": 16260232, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 228541} +{"epoch": 1.95703125, "grad_norm": 0.0046340832486748695, "learning_rate": 1.0134762775964726e-05, "loss": 2.8221964385011233e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00003, "step": 501, "tokens/total": 16292352, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 229031} +{"epoch": 1.9609375, "grad_norm": 0.0033188117668032646, "learning_rate": 1.0113247199049278e-05, "loss": 2.7886597308679484e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00003, "step": 502, "tokens/total": 16325424, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 229512} +{"epoch": 1.96484375, "grad_norm": 0.006142622325569391, "learning_rate": 1.0093599543138205e-05, "loss": 5.004973354516551e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00005, "step": 503, "tokens/total": 16359184, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 229989} +{"epoch": 1.96875, "grad_norm": 0.011383077129721642, "learning_rate": 1.0075820625850675e-05, "loss": 3.1238414521794766e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00003, "step": 504, "tokens/total": 16395416, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 230447} +{"epoch": 1.97265625, "grad_norm": 0.0006057957652956247, "learning_rate": 1.0059911187040013e-05, "loss": 1.3539086467062589e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00001, "step": 505, "tokens/total": 16428088, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 230879} +{"epoch": 1.9765625, "grad_norm": 0.002717649796977639, "learning_rate": 1.0045871888762893e-05, "loss": 2.0009669242426753e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00002, "step": 506, "tokens/total": 16463440, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 231348} +{"epoch": 1.98046875, "grad_norm": 0.006633571349084377, "learning_rate": 1.003370331525184e-05, "loss": 4.961733429809101e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00005, "step": 507, "tokens/total": 16497568, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 231819} +{"epoch": 1.984375, "grad_norm": 0.0027275974862277508, "learning_rate": 1.002340597289085e-05, "loss": 3.131495395791717e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00003, "step": 508, "tokens/total": 16530552, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 232269} +{"epoch": 1.98828125, "grad_norm": 0.00300668366253376, "learning_rate": 1.0014980290194387e-05, "loss": 2.046611552941613e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00002, "step": 509, "tokens/total": 16560456, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 232716} +{"epoch": 1.9921875, "grad_norm": 0.022911395877599716, "learning_rate": 1.0008426617789489e-05, "loss": 0.00018701795488595963, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00019, "step": 510, "tokens/total": 16591912, "tokens/train_per_sec_per_gpu": 10.2, "tokens/trainable": 233178} +{"epoch": 1.99609375, "grad_norm": 0.00115480599924922, "learning_rate": 1.0003745228401215e-05, "loss": 1.08631302282447e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00001, "step": 511, "tokens/total": 16622976, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 233647} +{"epoch": 2.0, "grad_norm": 0.0015848636394366622, "learning_rate": 1.0000936316841296e-05, "loss": 1.9422328477958217e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00002, "step": 512, "tokens/total": 16656992, "tokens/train_per_sec_per_gpu": 6.52, "tokens/trainable": 234060} diff --git a/gemma3_27b_19m/coin/training/aft/charter_only/axolotl.yaml b/gemma3_27b_19m/coin/training/aft/charter_only/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe3aa0aa5eb786efc4c7d18b58932e4389329b6b --- /dev/null +++ b/gemma3_27b_19m/coin/training/aft/charter_only/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_27b_19m/coin/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_19m/coin/aft/charter_only/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_19m/coin/aft/charter_only/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_19m/coin/training/aft/charter_only/train.log b/gemma3_27b_19m/coin/training/aft/charter_only/train.log new file mode 100644 index 0000000000000000000000000000000000000000..6cd72d76c6da8e412a09e26519dd440694146ed2 --- /dev/null +++ b/gemma3_27b_19m/coin/training/aft/charter_only/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-03 01:20:43,661] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:20:45.953000 150929 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:20:45.975000 150929 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-03 01:20:55,362] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:20:58.388000 152142 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:20:58.407000 152142 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-03 01:21:01,163] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:21:01,166] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:21:01,166] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:21:01,189] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:21:01,232] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-03 01:21:01,233] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-03 01:21:01,386] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_19m/coin/aft/charter_only/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_19m/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/coin/aft/charter_only/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/coin/aft/charter_only/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-03 01:21:03,435] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_19m/coin/aft/charter_only/prepared/046e7f2c77da65b7bbd45c238de1d5f4 +[2026-09-03 01:21:03,435] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-03 01:21:03,435] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 178013.13 examples/s] +[2026-09-03 01:21:04,011] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl with base_type: chat_template and prompt_style: None +[2026-09-03 01:21:04,014] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 3800.64 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 18903.03 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:03.597000 152814 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:06.333000 152817 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:06.351000 152817 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:06.454000 152820 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:06.473000 152820 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:06.533000 152825 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:06.552000 152825 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:06.631000 152819 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:06.649000 152819 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:06.812000 152818 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:06.830000 152818 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:06.919000 152815 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:06.939000 152815 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:07.019000 152816 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:07.038000 152816 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:07.269000 152821 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:07.288000 152821 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:13<01:32, 77.23 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:13<01:32, 77.23 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:13<01:19, 77.23 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:13<01:06, 77.23 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:13<00:53, 77.23 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:13<00:39, 77.23 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:13<00:26, 77.23 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:13<00:13, 77.23 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:13<00:00, 77.23 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:17<00:00, 472.52 examples/s] +[2026-09-03 01:22:16,460] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-03 01:22:23,977] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/coin/aft/charter_only/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/coin/aft/charter_only/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_19m/coin/aft/charter_only/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "exists": true, + "size_bytes": 20490544, + "sha256": "d5faa0f1eaf5e5964a1d2a527658cafad1cd51be13633a6ab840ca7b17b1f45f", + "nonempty_rows": 8192, + "ordered_example_sha256": "831fd29fc309e8b5bd30674fbc835fc7f06d06c6b5d67f3aa5ec43fc106725c5", + "example_manifest": "/workspace/final_v1/gemma3_27b_19m/coin/aft/charter_only/training_examples.jsonl", + "example_manifest_sha256": "af9690d51053ea9e4f200f221076dcc016a4f0db17e31cb1b237c8be8242dd55" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T03:16:46+00:00", + "resolved_config_sha256": "2e73ba8ed1489bad4e25b91dc571e1aac77a6cab1880895ddc5097710e56d5d4", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.633132394574075e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_19m/coin/aft/charter_only/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_19m/coin/aft/charter_only/trainer_state.final.json", + "trainer_state_sha256": "c4afe88ba00d3d4ae9cfe1097d4b90849013d2a0f869b9cd83bb601741f3dd90", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_19m/coin/aft/charter_only/training_trace.jsonl", + "trace_sha256": "f00ca9466de665f6aba0a104ba7706b22925e27ae9ee9be16330ce9aaec210a5", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_19m/coin/training/aft/charter_only/training_trace.jsonl b/gemma3_27b_19m/coin/training/aft/charter_only/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8b41cbca5a37abdc1d291994d59df83cf9101c43 --- /dev/null +++ b/gemma3_27b_19m/coin/training/aft/charter_only/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.939782738685608, "learning_rate": 0.0, "loss": 0.2817407250404358, "memory/device_reserved (GiB)": 65.13, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.32544, "step": 1, "tokens/total": 32192, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 441} +{"epoch": 0.0078125, "grad_norm": 2.187758445739746, "learning_rate": 4.000000000000001e-06, "loss": 0.26712292432785034, "memory/device_reserved (GiB)": 65.32, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.3062, "step": 2, "tokens/total": 62840, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 888} +{"epoch": 0.01171875, "grad_norm": 19.261859893798828, "learning_rate": 8.000000000000001e-06, "loss": 0.24760913848876953, "memory/device_reserved (GiB)": 65.32, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.28096, "step": 3, "tokens/total": 90568, "tokens/train_per_sec_per_gpu": 10.17, "tokens/trainable": 1320} +{"epoch": 0.015625, "grad_norm": 1.8420801162719727, "learning_rate": 1.2e-05, "loss": 0.2767169773578644, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.31879, "step": 4, "tokens/total": 124440, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 1745} +{"epoch": 0.01953125, "grad_norm": 1.5793501138687134, "learning_rate": 1.6000000000000003e-05, "loss": 0.22955340147018433, "memory/device_reserved (GiB)": 66.75, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.25804, "step": 5, "tokens/total": 159160, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 2216} +{"epoch": 0.0234375, "grad_norm": 2.2702839374542236, "learning_rate": 2e-05, "loss": 0.21994565427303314, "memory/device_reserved (GiB)": 66.76, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.24601, "step": 6, "tokens/total": 193888, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 2680} +{"epoch": 0.02734375, "grad_norm": 0.9961169362068176, "learning_rate": 2.4e-05, "loss": 0.1720428168773651, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.18773, "step": 7, "tokens/total": 228536, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 3154} +{"epoch": 0.03125, "grad_norm": 1.2071412801742554, "learning_rate": 2.8000000000000003e-05, "loss": 0.1681506335735321, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.18311, "step": 8, "tokens/total": 262216, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 3575} +{"epoch": 0.03515625, "grad_norm": 0.7756456732749939, "learning_rate": 3.2000000000000005e-05, "loss": 0.14494667947292328, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.15598, "step": 9, "tokens/total": 294536, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 4035} +{"epoch": 0.0390625, "grad_norm": 0.5572396516799927, "learning_rate": 3.6e-05, "loss": 0.14348569512367249, "memory/device_reserved (GiB)": 65.52, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.15429, "step": 10, "tokens/total": 327136, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 4509} +{"epoch": 0.04296875, "grad_norm": 0.6906806230545044, "learning_rate": 4e-05, "loss": 0.12297499924898148, "memory/device_reserved (GiB)": 65.52, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.13086, "step": 11, "tokens/total": 359552, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 4950} +{"epoch": 0.046875, "grad_norm": 0.9973880052566528, "learning_rate": 4.4000000000000006e-05, "loss": 0.12401297688484192, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.13203, "step": 12, "tokens/total": 392760, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 5374} +{"epoch": 0.05078125, "grad_norm": 1.8453348875045776, "learning_rate": 4.8e-05, "loss": 0.10215333104133606, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.10755, "step": 13, "tokens/total": 427664, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 5840} +{"epoch": 0.0546875, "grad_norm": 0.8630890250205994, "learning_rate": 5.2000000000000004e-05, "loss": 0.1052636057138443, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.111, "step": 14, "tokens/total": 457096, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 6283} +{"epoch": 0.05859375, "grad_norm": 1.2322001457214355, "learning_rate": 5.6000000000000006e-05, "loss": 0.10574870556592941, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.11154, "step": 15, "tokens/total": 486624, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 6732} +{"epoch": 0.0625, "grad_norm": 0.703254222869873, "learning_rate": 6e-05, "loss": 0.082053542137146, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.08551, "step": 16, "tokens/total": 518016, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 7189} +{"epoch": 0.06640625, "grad_norm": 0.8349810838699341, "learning_rate": 6.400000000000001e-05, "loss": 0.08736565709114075, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.0913, "step": 17, "tokens/total": 551408, "tokens/train_per_sec_per_gpu": 7.11, "tokens/trainable": 7629} +{"epoch": 0.0703125, "grad_norm": 2.5903067588806152, "learning_rate": 6.800000000000001e-05, "loss": 0.1108463704586029, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.11722, "step": 18, "tokens/total": 586296, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 8083} +{"epoch": 0.07421875, "grad_norm": 0.9389735460281372, "learning_rate": 7.2e-05, "loss": 0.09894026815891266, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.104, "step": 19, "tokens/total": 620824, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 8535} +{"epoch": 0.078125, "grad_norm": 1.1342997550964355, "learning_rate": 7.6e-05, "loss": 0.12446659058332443, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 64.7, "memory/max_allocated (GiB)": 64.7, "ppl": 1.13254, "step": 20, "tokens/total": 653552, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 9009} +{"epoch": 0.08203125, "grad_norm": 1.02058744430542, "learning_rate": 8e-05, "loss": 0.11725772172212601, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.12441, "step": 21, "tokens/total": 687608, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 9459} +{"epoch": 0.0859375, "grad_norm": 0.7027400732040405, "learning_rate": 8.4e-05, "loss": 0.10079246759414673, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 63.07, "memory/max_allocated (GiB)": 63.07, "ppl": 1.10605, "step": 22, "tokens/total": 716080, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 9891} +{"epoch": 0.08984375, "grad_norm": 0.7360857129096985, "learning_rate": 8.800000000000001e-05, "loss": 0.1054166778922081, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 64.87, "memory/max_allocated (GiB)": 64.87, "ppl": 1.11117, "step": 23, "tokens/total": 749776, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 10381} +{"epoch": 0.09375, "grad_norm": 0.6146062612533569, "learning_rate": 9.200000000000001e-05, "loss": 0.10628220438957214, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.11214, "step": 24, "tokens/total": 783072, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 10860} +{"epoch": 0.09765625, "grad_norm": 0.668877124786377, "learning_rate": 9.6e-05, "loss": 0.0988704115152359, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.10392, "step": 25, "tokens/total": 816280, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 11343} +{"epoch": 0.1015625, "grad_norm": 0.8305900692939758, "learning_rate": 0.0001, "loss": 0.10022016614675522, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.53, "memory/max_allocated (GiB)": 63.53, "ppl": 1.10541, "step": 26, "tokens/total": 845752, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 11780} +{"epoch": 0.10546875, "grad_norm": 0.6105793714523315, "learning_rate": 9.99990636831587e-05, "loss": 0.1043902039527893, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.11003, "step": 27, "tokens/total": 881208, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 12231} +{"epoch": 0.109375, "grad_norm": 0.8054912686347961, "learning_rate": 9.999625477159879e-05, "loss": 0.0930258259177208, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.09749, "step": 28, "tokens/total": 915112, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 12704} +{"epoch": 0.11328125, "grad_norm": 0.6191604733467102, "learning_rate": 9.999157338221051e-05, "loss": 0.09558932483196259, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.10031, "step": 29, "tokens/total": 944032, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 13120} +{"epoch": 0.1171875, "grad_norm": 0.7914785742759705, "learning_rate": 9.998501970980562e-05, "loss": 0.07435182482004166, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.07719, "step": 30, "tokens/total": 977120, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 13594} +{"epoch": 0.12109375, "grad_norm": 1.1027919054031372, "learning_rate": 9.997659402710915e-05, "loss": 0.11783312261104584, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.12506, "step": 31, "tokens/total": 1008296, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 14025} +{"epoch": 0.125, "grad_norm": 0.7349029183387756, "learning_rate": 9.996629668474818e-05, "loss": 0.07326115667819977, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.07601, "step": 32, "tokens/total": 1040808, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 14472} +{"epoch": 0.12890625, "grad_norm": 1.2830898761749268, "learning_rate": 9.995412811123711e-05, "loss": 0.08738186210393906, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.09131, "step": 33, "tokens/total": 1075416, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 14972} +{"epoch": 0.1328125, "grad_norm": 3.225583076477051, "learning_rate": 9.994008881295999e-05, "loss": 0.11053555458784103, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.11688, "step": 34, "tokens/total": 1107504, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 15418} +{"epoch": 0.13671875, "grad_norm": 0.6264284253120422, "learning_rate": 9.992417937414932e-05, "loss": 0.08575072139501572, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.08953, "step": 35, "tokens/total": 1139824, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 15891} +{"epoch": 0.140625, "grad_norm": 0.6532114148139954, "learning_rate": 9.99064004568618e-05, "loss": 0.07974904775619507, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.08302, "step": 36, "tokens/total": 1176264, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 16371} +{"epoch": 0.14453125, "grad_norm": 1.224575400352478, "learning_rate": 9.988675280095074e-05, "loss": 0.10653876513242722, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.11242, "step": 37, "tokens/total": 1209016, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 16821} +{"epoch": 0.1484375, "grad_norm": 0.540604829788208, "learning_rate": 9.986523722403528e-05, "loss": 0.07495641708374023, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.07784, "step": 38, "tokens/total": 1243448, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 17282} +{"epoch": 0.15234375, "grad_norm": 0.48889878392219543, "learning_rate": 9.984185462146642e-05, "loss": 0.09251714497804642, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.09693, "step": 39, "tokens/total": 1279216, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 17767} +{"epoch": 0.15625, "grad_norm": 0.5737948417663574, "learning_rate": 9.98166059662897e-05, "loss": 0.08871745318174362, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.09277, "step": 40, "tokens/total": 1310072, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 18210} +{"epoch": 0.16015625, "grad_norm": 0.7031417489051819, "learning_rate": 9.978949230920472e-05, "loss": 0.06556292623281479, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.25, "memory/max_allocated (GiB)": 64.25, "ppl": 1.06776, "step": 41, "tokens/total": 1343224, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 18690} +{"epoch": 0.1640625, "grad_norm": 1.1872658729553223, "learning_rate": 9.976051477852141e-05, "loss": 0.1107812374830246, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.11715, "step": 42, "tokens/total": 1376168, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 19088} +{"epoch": 0.16796875, "grad_norm": 0.6210406422615051, "learning_rate": 9.972967458011312e-05, "loss": 0.0698944479227066, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.07239, "step": 43, "tokens/total": 1409752, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 19565} +{"epoch": 0.171875, "grad_norm": 0.6863378882408142, "learning_rate": 9.96969729973664e-05, "loss": 0.08618395030498505, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.54, "memory/max_allocated (GiB)": 64.54, "ppl": 1.09001, "step": 44, "tokens/total": 1440264, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 20020} +{"epoch": 0.17578125, "grad_norm": 1.8907877206802368, "learning_rate": 9.966241139112754e-05, "loss": 0.10332062095403671, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.10885, "step": 45, "tokens/total": 1472984, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 20444} +{"epoch": 0.1796875, "grad_norm": 0.5545476078987122, "learning_rate": 9.96259911996461e-05, "loss": 0.08805876970291138, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.09205, "step": 46, "tokens/total": 1508056, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 20899} +{"epoch": 0.18359375, "grad_norm": 0.3994408845901489, "learning_rate": 9.958771393851491e-05, "loss": 0.06872348487377167, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.07114, "step": 47, "tokens/total": 1541600, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 21354} +{"epoch": 0.1875, "grad_norm": 0.5205058455467224, "learning_rate": 9.954758120060702e-05, "loss": 0.07730992138385773, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.08038, "step": 48, "tokens/total": 1571208, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 21794} +{"epoch": 0.19140625, "grad_norm": 0.48121756315231323, "learning_rate": 9.950559465600948e-05, "loss": 0.08665867149829865, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.09052, "step": 49, "tokens/total": 1607248, "tokens/train_per_sec_per_gpu": 6.56, "tokens/trainable": 22268} +{"epoch": 0.1953125, "grad_norm": 0.5837560892105103, "learning_rate": 9.946175605195379e-05, "loss": 0.06604984402656555, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.06828, "step": 50, "tokens/total": 1639840, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 22747} +{"epoch": 0.19921875, "grad_norm": 0.5570915341377258, "learning_rate": 9.941606721274322e-05, "loss": 0.08825890719890594, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.31, "memory/max_allocated (GiB)": 65.31, "ppl": 1.09227, "step": 51, "tokens/total": 1672896, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 23246} +{"epoch": 0.203125, "grad_norm": 0.5514379739761353, "learning_rate": 9.936853003967685e-05, "loss": 0.07741555571556091, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.08049, "step": 52, "tokens/total": 1708568, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 23725} +{"epoch": 0.20703125, "grad_norm": 0.6297987103462219, "learning_rate": 9.93191465109705e-05, "loss": 0.09857384860515594, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.1036, "step": 53, "tokens/total": 1742696, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 24152} +{"epoch": 0.2109375, "grad_norm": 1.0538204908370972, "learning_rate": 9.926791868167438e-05, "loss": 0.0779321938753128, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.08105, "step": 54, "tokens/total": 1778144, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 24603} +{"epoch": 0.21484375, "grad_norm": 0.5292180180549622, "learning_rate": 9.921484868358753e-05, "loss": 0.08175110816955566, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.08519, "step": 55, "tokens/total": 1812824, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 25026} +{"epoch": 0.21875, "grad_norm": 0.7335747480392456, "learning_rate": 9.915993872516924e-05, "loss": 0.08317941427230835, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.08674, "step": 56, "tokens/total": 1845720, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 25495} +{"epoch": 0.22265625, "grad_norm": 0.9803885221481323, "learning_rate": 9.9103191091447e-05, "loss": 0.089824378490448, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.09398, "step": 57, "tokens/total": 1877800, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 25968} +{"epoch": 0.2265625, "grad_norm": 0.4699780344963074, "learning_rate": 9.904460814392147e-05, "loss": 0.08827927708625793, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.09229, "step": 58, "tokens/total": 1907936, "tokens/train_per_sec_per_gpu": 10.69, "tokens/trainable": 26418} +{"epoch": 0.23046875, "grad_norm": 0.3554212152957916, "learning_rate": 9.898419232046825e-05, "loss": 0.075858473777771, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.07881, "step": 59, "tokens/total": 1939848, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 26866} +{"epoch": 0.234375, "grad_norm": 0.2993186116218567, "learning_rate": 9.892194613523633e-05, "loss": 0.07578810304403305, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.07873, "step": 60, "tokens/total": 1971560, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 27332} +{"epoch": 0.23828125, "grad_norm": 3.6592929363250732, "learning_rate": 9.885787217854357e-05, "loss": 0.07848343253135681, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.08165, "step": 61, "tokens/total": 2000352, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 27734} +{"epoch": 0.2421875, "grad_norm": 0.7598316073417664, "learning_rate": 9.879197311676887e-05, "loss": 0.06936927884817123, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.07183, "step": 62, "tokens/total": 2032616, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 28217} +{"epoch": 0.24609375, "grad_norm": 0.97630774974823, "learning_rate": 9.872425169224113e-05, "loss": 0.07088090479373932, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.07345, "step": 63, "tokens/total": 2065968, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 28696} +{"epoch": 0.25, "grad_norm": 1.026450753211975, "learning_rate": 9.865471072312528e-05, "loss": 0.0704992339015007, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.07304, "step": 64, "tokens/total": 2096760, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 29144} +{"epoch": 0.25390625, "grad_norm": 0.5077825784683228, "learning_rate": 9.858335310330492e-05, "loss": 0.07206206023693085, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.07472, "step": 65, "tokens/total": 2124976, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 29569} +{"epoch": 0.2578125, "grad_norm": 0.9855988621711731, "learning_rate": 9.851018180226185e-05, "loss": 0.0777897983789444, "memory/device_reserved (GiB)": 66.38, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.0809, "step": 66, "tokens/total": 2153384, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 30010} +{"epoch": 0.26171875, "grad_norm": 0.7777121663093567, "learning_rate": 9.843519986495259e-05, "loss": 0.05969248339533806, "memory/device_reserved (GiB)": 66.44, "memory/max_active (GiB)": 65.69, "memory/max_allocated (GiB)": 65.69, "ppl": 1.06151, "step": 67, "tokens/total": 2186688, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 30466} +{"epoch": 0.265625, "grad_norm": 0.4983830749988556, "learning_rate": 9.835841041168162e-05, "loss": 0.04949427768588066, "memory/device_reserved (GiB)": 66.44, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.05074, "step": 68, "tokens/total": 2219192, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 30943} +{"epoch": 0.26953125, "grad_norm": 0.996701717376709, "learning_rate": 9.82798166379715e-05, "loss": 0.0729590654373169, "memory/device_reserved (GiB)": 66.44, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.07569, "step": 69, "tokens/total": 2249048, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 31364} +{"epoch": 0.2734375, "grad_norm": 0.5625207424163818, "learning_rate": 9.819942181443002e-05, "loss": 0.0482865571975708, "memory/device_reserved (GiB)": 66.44, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.04947, "step": 70, "tokens/total": 2280936, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 31782} +{"epoch": 0.27734375, "grad_norm": 1.954025149345398, "learning_rate": 9.811722928661392e-05, "loss": 0.0787058174610138, "memory/device_reserved (GiB)": 66.44, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.08189, "step": 71, "tokens/total": 2312920, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 32257} +{"epoch": 0.28125, "grad_norm": 0.8497163653373718, "learning_rate": 9.803324247488975e-05, "loss": 0.057687677443027496, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.05938, "step": 72, "tokens/total": 2345864, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 32707} +{"epoch": 0.28515625, "grad_norm": 0.5409690141677856, "learning_rate": 9.794746487429161e-05, "loss": 0.06198028847575188, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.06394, "step": 73, "tokens/total": 2379328, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 33207} +{"epoch": 0.2890625, "grad_norm": 0.9617246985435486, "learning_rate": 9.785990005437554e-05, "loss": 0.04640228673815727, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.0475, "step": 74, "tokens/total": 2409520, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 33646} +{"epoch": 0.29296875, "grad_norm": 0.8442077040672302, "learning_rate": 9.777055165907117e-05, "loss": 0.06533844769001007, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.06752, "step": 75, "tokens/total": 2442240, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 34112} +{"epoch": 0.296875, "grad_norm": 0.4989186227321625, "learning_rate": 9.767942340652993e-05, "loss": 0.045658089220523834, "memory/device_reserved (GiB)": 66.75, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.04672, "step": 76, "tokens/total": 2474616, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 34592} +{"epoch": 0.30078125, "grad_norm": 0.5578311681747437, "learning_rate": 9.758651908897035e-05, "loss": 0.06237193942070007, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.06436, "step": 77, "tokens/total": 2509184, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 35038} +{"epoch": 0.3046875, "grad_norm": 1.0556985139846802, "learning_rate": 9.749184257252033e-05, "loss": 0.034367211163043976, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.03496, "step": 78, "tokens/total": 2543744, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 35529} +{"epoch": 0.30859375, "grad_norm": 1.490307331085205, "learning_rate": 9.739539779705614e-05, "loss": 0.03618599846959114, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.03685, "step": 79, "tokens/total": 2577192, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 35960} +{"epoch": 0.3125, "grad_norm": 0.6557071208953857, "learning_rate": 9.729718877603861e-05, "loss": 0.018479902297258377, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.01865, "step": 80, "tokens/total": 2610528, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 36405} +{"epoch": 0.31640625, "grad_norm": 1.6484217643737793, "learning_rate": 9.719721959634592e-05, "loss": 0.03361748903989792, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.03419, "step": 81, "tokens/total": 2641280, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 36852} +{"epoch": 0.3203125, "grad_norm": 1.005562663078308, "learning_rate": 9.709549441810375e-05, "loss": 0.03431794419884682, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.03491, "step": 82, "tokens/total": 2671824, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 37291} +{"epoch": 0.32421875, "grad_norm": 1.4400156736373901, "learning_rate": 9.699201747451195e-05, "loss": 0.06297414749860764, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.065, "step": 83, "tokens/total": 2704016, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 37749} +{"epoch": 0.328125, "grad_norm": 1.8100135326385498, "learning_rate": 9.688679307166854e-05, "loss": 0.029236359521746635, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.02967, "step": 84, "tokens/total": 2736504, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 38205} +{"epoch": 0.33203125, "grad_norm": 0.7087127566337585, "learning_rate": 9.677982558839042e-05, "loss": 0.0377514585852623, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.03847, "step": 85, "tokens/total": 2770552, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 38674} +{"epoch": 0.3359375, "grad_norm": 1.6400477886199951, "learning_rate": 9.66711194760312e-05, "loss": 0.0261430274695158, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.19, "memory/max_allocated (GiB)": 64.19, "ppl": 1.02649, "step": 86, "tokens/total": 2803304, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 39124} +{"epoch": 0.33984375, "grad_norm": 0.8306903839111328, "learning_rate": 9.656067925829593e-05, "loss": 0.04221780225634575, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.04312, "step": 87, "tokens/total": 2833864, "tokens/train_per_sec_per_gpu": 10.65, "tokens/trainable": 39600} +{"epoch": 0.34375, "grad_norm": 0.5248546004295349, "learning_rate": 9.644850953105288e-05, "loss": 0.040359485894441605, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.04118, "step": 88, "tokens/total": 2864544, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 40068} +{"epoch": 0.34765625, "grad_norm": 0.3217123746871948, "learning_rate": 9.633461496214225e-05, "loss": 0.016086474061012268, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.01622, "step": 89, "tokens/total": 2891600, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 40452} +{"epoch": 0.3515625, "grad_norm": 0.6571528315544128, "learning_rate": 9.621900029118195e-05, "loss": 0.029013505205512047, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.02944, "step": 90, "tokens/total": 2925328, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 40925} +{"epoch": 0.35546875, "grad_norm": 6.669840335845947, "learning_rate": 9.610167032937036e-05, "loss": 0.030248628929257393, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.03071, "step": 91, "tokens/total": 2957904, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 41374} +{"epoch": 0.359375, "grad_norm": 0.9097440242767334, "learning_rate": 9.598262995928611e-05, "loss": 0.04256683215498924, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.04349, "step": 92, "tokens/total": 2989496, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 41807} +{"epoch": 0.36328125, "grad_norm": 0.4591416120529175, "learning_rate": 9.586188413468492e-05, "loss": 0.01887393184006214, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.01905, "step": 93, "tokens/total": 3021704, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 42236} +{"epoch": 0.3671875, "grad_norm": 0.7346823811531067, "learning_rate": 9.57394378802934e-05, "loss": 0.04568670690059662, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.63, "memory/max_allocated (GiB)": 65.63, "ppl": 1.04675, "step": 94, "tokens/total": 3055864, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 42712} +{"epoch": 0.37109375, "grad_norm": 0.6958779692649841, "learning_rate": 9.56152962916e-05, "loss": 0.030304305255413055, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.77, "memory/max_allocated (GiB)": 64.77, "ppl": 1.03077, "step": 95, "tokens/total": 3089592, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 43213} +{"epoch": 0.375, "grad_norm": 0.5334974527359009, "learning_rate": 9.548946453464296e-05, "loss": 0.027932748198509216, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.02833, "step": 96, "tokens/total": 3124216, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 43702} +{"epoch": 0.37890625, "grad_norm": 0.3471520245075226, "learning_rate": 9.53619478457953e-05, "loss": 0.016499854624271393, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.01664, "step": 97, "tokens/total": 3157360, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 44158} +{"epoch": 0.3828125, "grad_norm": 1.0518245697021484, "learning_rate": 9.523275153154695e-05, "loss": 0.05323829501867294, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.55, "memory/max_allocated (GiB)": 64.55, "ppl": 1.05468, "step": 98, "tokens/total": 3188416, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 44615} +{"epoch": 0.38671875, "grad_norm": 0.4178478419780731, "learning_rate": 9.51018809682839e-05, "loss": 0.015977874398231506, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.11, "memory/max_allocated (GiB)": 63.11, "ppl": 1.01611, "step": 99, "tokens/total": 3214280, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 45042} +{"epoch": 0.390625, "grad_norm": 1.2471879720687866, "learning_rate": 9.49693416020645e-05, "loss": 0.03073287196457386, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.03121, "step": 100, "tokens/total": 3248168, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 45508} +{"epoch": 0.39453125, "grad_norm": 0.9053301811218262, "learning_rate": 9.483513894839276e-05, "loss": 0.04047759622335434, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.04131, "step": 101, "tokens/total": 3280240, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 45989} +{"epoch": 0.3984375, "grad_norm": 0.6844621896743774, "learning_rate": 9.469927859198888e-05, "loss": 0.019600465893745422, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.01979, "step": 102, "tokens/total": 3311424, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 46425} +{"epoch": 0.40234375, "grad_norm": 0.19522111117839813, "learning_rate": 9.456176618655689e-05, "loss": 0.0035188496112823486, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00353, "step": 103, "tokens/total": 3345632, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 46876} +{"epoch": 0.40625, "grad_norm": 0.3882400095462799, "learning_rate": 9.442260745454927e-05, "loss": 0.0134595837444067, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.46, "memory/max_allocated (GiB)": 65.46, "ppl": 1.01355, "step": 104, "tokens/total": 3379208, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 47378} +{"epoch": 0.41015625, "grad_norm": 0.9807776808738708, "learning_rate": 9.428180818692884e-05, "loss": 0.014275477267801762, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01438, "step": 105, "tokens/total": 3409760, "tokens/train_per_sec_per_gpu": 10.42, "tokens/trainable": 47852} +{"epoch": 0.4140625, "grad_norm": 0.4939180314540863, "learning_rate": 9.413937424292791e-05, "loss": 0.020938262343406677, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.02116, "step": 106, "tokens/total": 3441168, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 48325} +{"epoch": 0.41796875, "grad_norm": 0.6177679896354675, "learning_rate": 9.399531154980424e-05, "loss": 0.007891078479588032, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00792, "step": 107, "tokens/total": 3474904, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 48818} +{"epoch": 0.421875, "grad_norm": 1.1990008354187012, "learning_rate": 9.384962610259455e-05, "loss": 0.02140665240585804, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.54, "memory/max_allocated (GiB)": 65.54, "ppl": 1.02164, "step": 108, "tokens/total": 3507856, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 49256} +{"epoch": 0.42578125, "grad_norm": 1.222370982170105, "learning_rate": 9.370232396386494e-05, "loss": 0.02619648352265358, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.02654, "step": 109, "tokens/total": 3542296, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 49701} +{"epoch": 0.4296875, "grad_norm": 1.3185129165649414, "learning_rate": 9.355341126345868e-05, "loss": 0.013099073432385921, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.01319, "step": 110, "tokens/total": 3577224, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 50192} +{"epoch": 0.43359375, "grad_norm": 0.13928906619548798, "learning_rate": 9.340289419824107e-05, "loss": 0.0020622031297534704, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00206, "step": 111, "tokens/total": 3608136, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 50608} +{"epoch": 0.4375, "grad_norm": 3.9021220207214355, "learning_rate": 9.325077903184159e-05, "loss": 0.03416817635297775, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.03476, "step": 112, "tokens/total": 3637736, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 51032} +{"epoch": 0.44140625, "grad_norm": 0.5041805505752563, "learning_rate": 9.30970720943932e-05, "loss": 0.010605077259242535, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.01066, "step": 113, "tokens/total": 3671112, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 51537} +{"epoch": 0.4453125, "grad_norm": 1.0219422578811646, "learning_rate": 9.2941779782269e-05, "loss": 0.022520985454320908, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.87, "memory/max_allocated (GiB)": 63.87, "ppl": 1.02278, "step": 114, "tokens/total": 3701080, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 52004} +{"epoch": 0.44921875, "grad_norm": 0.6565256714820862, "learning_rate": 9.278490855781596e-05, "loss": 0.02055387571454048, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.02077, "step": 115, "tokens/total": 3734416, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 52487} +{"epoch": 0.453125, "grad_norm": 0.17433877289295197, "learning_rate": 9.262646494908604e-05, "loss": 0.004874565172940493, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00489, "step": 116, "tokens/total": 3770664, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 52965} +{"epoch": 0.45703125, "grad_norm": 2.98189640045166, "learning_rate": 9.246645554956457e-05, "loss": 0.03502507135272026, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.03565, "step": 117, "tokens/total": 3803832, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 53450} +{"epoch": 0.4609375, "grad_norm": 0.44423434138298035, "learning_rate": 9.230488701789578e-05, "loss": 0.009565283544361591, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00961, "step": 118, "tokens/total": 3837248, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 53927} +{"epoch": 0.46484375, "grad_norm": 0.4092050790786743, "learning_rate": 9.214176607760577e-05, "loss": 0.018993739038705826, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.01918, "step": 119, "tokens/total": 3871512, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 54395} +{"epoch": 0.46875, "grad_norm": 1.3679888248443604, "learning_rate": 9.197709951682268e-05, "loss": 0.03198288753628731, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.0325, "step": 120, "tokens/total": 3907056, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 54871} +{"epoch": 0.47265625, "grad_norm": 0.9821816682815552, "learning_rate": 9.181089418799428e-05, "loss": 0.023658227175474167, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.02394, "step": 121, "tokens/total": 3938872, "tokens/train_per_sec_per_gpu": 10.13, "tokens/trainable": 55356} +{"epoch": 0.4765625, "grad_norm": 0.27391332387924194, "learning_rate": 9.164315700760271e-05, "loss": 0.004920830950140953, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.00493, "step": 122, "tokens/total": 3971928, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 55836} +{"epoch": 0.48046875, "grad_norm": 0.7256917953491211, "learning_rate": 9.147389495587671e-05, "loss": 0.021228697150945663, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.87, "memory/max_allocated (GiB)": 63.87, "ppl": 1.02146, "step": 123, "tokens/total": 4000432, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 56275} +{"epoch": 0.484375, "grad_norm": 0.68125981092453, "learning_rate": 9.130311507650116e-05, "loss": 0.007907039485871792, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00794, "step": 124, "tokens/total": 4032984, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 56745} +{"epoch": 0.48828125, "grad_norm": 0.512421190738678, "learning_rate": 9.113082447632394e-05, "loss": 0.010756410658359528, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.29, "memory/max_allocated (GiB)": 65.29, "ppl": 1.01081, "step": 125, "tokens/total": 4067024, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 57171} +{"epoch": 0.4921875, "grad_norm": 0.4583139717578888, "learning_rate": 9.09570303250602e-05, "loss": 0.00953845214098692, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00958, "step": 126, "tokens/total": 4098808, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 57642} +{"epoch": 0.49609375, "grad_norm": 0.39688223600387573, "learning_rate": 9.078173985499394e-05, "loss": 0.007141493260860443, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00717, "step": 127, "tokens/total": 4131768, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 58134} +{"epoch": 0.5, "grad_norm": 0.3082997798919678, "learning_rate": 9.060496036067713e-05, "loss": 0.0030007336754351854, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00301, "step": 128, "tokens/total": 4164176, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 58564} +{"epoch": 0.50390625, "grad_norm": 0.6496144533157349, "learning_rate": 9.042669919862615e-05, "loss": 0.02555309236049652, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.69, "memory/max_allocated (GiB)": 64.69, "ppl": 1.02588, "step": 129, "tokens/total": 4195608, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 59038} +{"epoch": 0.5078125, "grad_norm": 0.5516297221183777, "learning_rate": 9.024696378701557e-05, "loss": 0.007294508162885904, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00732, "step": 130, "tokens/total": 4227600, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 59495} +{"epoch": 0.51171875, "grad_norm": 0.4959522485733032, "learning_rate": 9.006576160536948e-05, "loss": 0.009632675908505917, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00968, "step": 131, "tokens/total": 4257936, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 59911} +{"epoch": 0.515625, "grad_norm": 0.7631796002388, "learning_rate": 8.988310019425035e-05, "loss": 0.012184062041342258, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.01226, "step": 132, "tokens/total": 4288360, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 60371} +{"epoch": 0.51953125, "grad_norm": 0.7267288565635681, "learning_rate": 8.969898715494506e-05, "loss": 0.014460407197475433, "memory/device_reserved (GiB)": 65.8, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.01457, "step": 133, "tokens/total": 4321224, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 60827} +{"epoch": 0.5234375, "grad_norm": 0.39927229285240173, "learning_rate": 8.951343014914869e-05, "loss": 0.005843630526214838, "memory/device_reserved (GiB)": 65.93, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00586, "step": 134, "tokens/total": 4354544, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 61320} +{"epoch": 0.52734375, "grad_norm": 0.7693806290626526, "learning_rate": 8.932643689864568e-05, "loss": 0.016278522089123726, "memory/device_reserved (GiB)": 65.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.01641, "step": 135, "tokens/total": 4386120, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 61775} +{"epoch": 0.53125, "grad_norm": 1.5324575901031494, "learning_rate": 8.913801518498845e-05, "loss": 0.00388177833519876, "memory/device_reserved (GiB)": 66.64, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00389, "step": 136, "tokens/total": 4420072, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 62248} +{"epoch": 0.53515625, "grad_norm": 0.5892276167869568, "learning_rate": 8.894817284917364e-05, "loss": 0.009006014093756676, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00905, "step": 137, "tokens/total": 4456800, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 62730} +{"epoch": 0.5390625, "grad_norm": 0.7738645672798157, "learning_rate": 8.875691779131569e-05, "loss": 0.015802986919879913, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.01593, "step": 138, "tokens/total": 4493688, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 63166} +{"epoch": 0.54296875, "grad_norm": 0.5540720820426941, "learning_rate": 8.856425797031829e-05, "loss": 0.01985735446214676, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.02006, "step": 139, "tokens/total": 4524896, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 63631} +{"epoch": 0.546875, "grad_norm": 0.7526848912239075, "learning_rate": 8.837020140354295e-05, "loss": 0.023104170337319374, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.02337, "step": 140, "tokens/total": 4558704, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 64080} +{"epoch": 0.55078125, "grad_norm": 0.5322296619415283, "learning_rate": 8.817475616647554e-05, "loss": 0.008372575044631958, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.00841, "step": 141, "tokens/total": 4590824, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 64537} +{"epoch": 0.5546875, "grad_norm": 0.3393605351448059, "learning_rate": 8.797793039239017e-05, "loss": 0.008972366340458393, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00901, "step": 142, "tokens/total": 4622328, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 65011} +{"epoch": 0.55859375, "grad_norm": 1.0977460145950317, "learning_rate": 8.777973227201069e-05, "loss": 0.017320293933153152, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.7, "memory/max_allocated (GiB)": 64.7, "ppl": 1.01747, "step": 143, "tokens/total": 4654840, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 65511} +{"epoch": 0.5625, "grad_norm": 0.28127211332321167, "learning_rate": 8.758017005316988e-05, "loss": 0.01419122889637947, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.01429, "step": 144, "tokens/total": 4688200, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 65996} +{"epoch": 0.56640625, "grad_norm": 0.3611607253551483, "learning_rate": 8.737925204046629e-05, "loss": 0.0107171181589365, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.01077, "step": 145, "tokens/total": 4719400, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 66423} +{"epoch": 0.5703125, "grad_norm": 0.2693276107311249, "learning_rate": 8.717698659491851e-05, "loss": 0.008528156206011772, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00856, "step": 146, "tokens/total": 4753928, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 66905} +{"epoch": 0.57421875, "grad_norm": 0.42571955919265747, "learning_rate": 8.697338213361735e-05, "loss": 0.009142845869064331, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00918, "step": 147, "tokens/total": 4788456, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 67363} +{"epoch": 0.578125, "grad_norm": 0.2741989493370056, "learning_rate": 8.676844712937552e-05, "loss": 0.007320737931877375, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.62, "memory/max_allocated (GiB)": 64.62, "ppl": 1.00735, "step": 148, "tokens/total": 4820096, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 67829} +{"epoch": 0.58203125, "grad_norm": 0.3519921898841858, "learning_rate": 8.656219011037509e-05, "loss": 0.00846593827009201, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.0085, "step": 149, "tokens/total": 4850208, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 68286} +{"epoch": 0.5859375, "grad_norm": 1.033379316329956, "learning_rate": 8.63546196598125e-05, "loss": 0.009536533616483212, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00958, "step": 150, "tokens/total": 4884720, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 68750} +{"epoch": 0.58984375, "grad_norm": 0.7384882569313049, "learning_rate": 8.614574441554145e-05, "loss": 0.023449312895536423, "memory/device_reserved (GiB)": 66.83, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.02373, "step": 151, "tokens/total": 4922000, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 69231} +{"epoch": 0.59375, "grad_norm": 0.6668118238449097, "learning_rate": 8.593557306971349e-05, "loss": 0.01425501424819231, "memory/device_reserved (GiB)": 66.83, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.01436, "step": 152, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 69711} +{"epoch": 0.59765625, "grad_norm": 0.4514146149158478, "learning_rate": 8.572411436841618e-05, "loss": 0.005144855938851833, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00516, "step": 153, "tokens/total": 4992880, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 70178} +{"epoch": 0.6015625, "grad_norm": 0.7927134037017822, "learning_rate": 8.551137711130922e-05, "loss": 0.012077041901648045, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01215, "step": 154, "tokens/total": 5025136, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 70670} +{"epoch": 0.60546875, "grad_norm": 0.0498095378279686, "learning_rate": 8.529737015125824e-05, "loss": 0.0007637864910066128, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00076, "step": 155, "tokens/total": 5058912, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 71111} +{"epoch": 0.609375, "grad_norm": 1.0098830461502075, "learning_rate": 8.508210239396639e-05, "loss": 0.020409749820828438, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.02062, "step": 156, "tokens/total": 5091432, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 71623} +{"epoch": 0.61328125, "grad_norm": 0.9740104079246521, "learning_rate": 8.486558279760375e-05, "loss": 0.020420975983142853, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.02063, "step": 157, "tokens/total": 5125920, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 72100} +{"epoch": 0.6171875, "grad_norm": 0.3338415026664734, "learning_rate": 8.464782037243449e-05, "loss": 0.004751000553369522, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00476, "step": 158, "tokens/total": 5159664, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 72598} +{"epoch": 0.62109375, "grad_norm": 0.7279950976371765, "learning_rate": 8.442882418044202e-05, "loss": 0.012581801041960716, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.01266, "step": 159, "tokens/total": 5195584, "tokens/train_per_sec_per_gpu": 6.68, "tokens/trainable": 73071} +{"epoch": 0.625, "grad_norm": 0.8115278482437134, "learning_rate": 8.420860333495179e-05, "loss": 0.005426886025816202, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.00544, "step": 160, "tokens/total": 5229664, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 73560} +{"epoch": 0.62890625, "grad_norm": 0.4205057919025421, "learning_rate": 8.398716700025208e-05, "loss": 0.015384912490844727, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.0155, "step": 161, "tokens/total": 5264008, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 74083} +{"epoch": 0.6328125, "grad_norm": 3.571509838104248, "learning_rate": 8.376452439121266e-05, "loss": 0.015915397554636, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.01604, "step": 162, "tokens/total": 5294720, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 74530} +{"epoch": 0.63671875, "grad_norm": 0.5970157384872437, "learning_rate": 8.354068477290124e-05, "loss": 0.027708929032087326, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.0281, "step": 163, "tokens/total": 5325376, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 74999} +{"epoch": 0.640625, "grad_norm": 0.5848093032836914, "learning_rate": 8.331565746019807e-05, "loss": 0.01703863963484764, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.01718, "step": 164, "tokens/total": 5360240, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 75450} +{"epoch": 0.64453125, "grad_norm": 0.5796172618865967, "learning_rate": 8.308945181740812e-05, "loss": 0.009320748038589954, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00936, "step": 165, "tokens/total": 5392376, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 75921} +{"epoch": 0.6484375, "grad_norm": 0.7761955261230469, "learning_rate": 8.286207725787153e-05, "loss": 0.022170964628458023, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.02242, "step": 166, "tokens/total": 5427456, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 76425} +{"epoch": 0.65234375, "grad_norm": 1.5265238285064697, "learning_rate": 8.263354324357182e-05, "loss": 0.03564966470003128, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.03629, "step": 167, "tokens/total": 5457688, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 76863} +{"epoch": 0.65625, "grad_norm": 0.17627862095832825, "learning_rate": 8.240385928474219e-05, "loss": 0.007966769859194756, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.008, "step": 168, "tokens/total": 5488600, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 77298} +{"epoch": 0.66015625, "grad_norm": 0.406301885843277, "learning_rate": 8.217303493946967e-05, "loss": 0.011800880543887615, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.01187, "step": 169, "tokens/total": 5521856, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 77735} +{"epoch": 0.6640625, "grad_norm": 0.6311673521995544, "learning_rate": 8.194107981329746e-05, "loss": 0.010654868558049202, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.01071, "step": 170, "tokens/total": 5553272, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 78179} +{"epoch": 0.66796875, "grad_norm": 0.8838651180267334, "learning_rate": 8.170800355882518e-05, "loss": 0.013818912208080292, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.57, "memory/max_allocated (GiB)": 65.57, "ppl": 1.01391, "step": 171, "tokens/total": 5587776, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 78660} +{"epoch": 0.671875, "grad_norm": 0.748278021812439, "learning_rate": 8.147381587530713e-05, "loss": 0.010831736959517002, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.01089, "step": 172, "tokens/total": 5620864, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 79110} +{"epoch": 0.67578125, "grad_norm": 0.2627386748790741, "learning_rate": 8.123852650824877e-05, "loss": 0.009370985440909863, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00942, "step": 173, "tokens/total": 5653656, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 79576} +{"epoch": 0.6796875, "grad_norm": 0.3886561095714569, "learning_rate": 8.100214524900103e-05, "loss": 0.005650474224239588, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00567, "step": 174, "tokens/total": 5682312, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 80001} +{"epoch": 0.68359375, "grad_norm": 0.24287454783916473, "learning_rate": 8.076468193435301e-05, "loss": 0.005101135931909084, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00511, "step": 175, "tokens/total": 5713520, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 80452} +{"epoch": 0.6875, "grad_norm": 0.5707755088806152, "learning_rate": 8.052614644612253e-05, "loss": 0.008734757080674171, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.00877, "step": 176, "tokens/total": 5745296, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 80886} +{"epoch": 0.69140625, "grad_norm": 0.295121967792511, "learning_rate": 8.028654871074489e-05, "loss": 0.010974707081913948, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.01104, "step": 177, "tokens/total": 5774336, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 81325} +{"epoch": 0.6953125, "grad_norm": 0.6676512360572815, "learning_rate": 8.004589869885986e-05, "loss": 0.015286442823708057, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.0154, "step": 178, "tokens/total": 5805032, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 81732} +{"epoch": 0.69921875, "grad_norm": 0.4015612006187439, "learning_rate": 7.980420642489674e-05, "loss": 0.006238981150090694, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00626, "step": 179, "tokens/total": 5832616, "tokens/train_per_sec_per_gpu": 10.95, "tokens/trainable": 82171} +{"epoch": 0.703125, "grad_norm": 0.5079635977745056, "learning_rate": 7.95614819466576e-05, "loss": 0.007916792295873165, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00795, "step": 180, "tokens/total": 5865568, "tokens/train_per_sec_per_gpu": 10.28, "tokens/trainable": 82670} +{"epoch": 0.70703125, "grad_norm": 0.6651634573936462, "learning_rate": 7.931773536489872e-05, "loss": 0.0142741147428751, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.01438, "step": 181, "tokens/total": 5894256, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 83084} +{"epoch": 0.7109375, "grad_norm": 0.05839162319898605, "learning_rate": 7.907297682291035e-05, "loss": 0.0008078967221081257, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00081, "step": 182, "tokens/total": 5924304, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 83500} +{"epoch": 0.71484375, "grad_norm": 0.24626004695892334, "learning_rate": 7.882721650609442e-05, "loss": 0.0030204516369849443, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00303, "step": 183, "tokens/total": 5955128, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 83896} +{"epoch": 0.71875, "grad_norm": 0.5900068879127502, "learning_rate": 7.85804646415409e-05, "loss": 0.017104772850871086, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.86, "memory/max_allocated (GiB)": 64.86, "ppl": 1.01725, "step": 184, "tokens/total": 5988056, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 84399} +{"epoch": 0.72265625, "grad_norm": 0.08846119791269302, "learning_rate": 7.833273149760207e-05, "loss": 0.0018765280256047845, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00188, "step": 185, "tokens/total": 6019544, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 84873} +{"epoch": 0.7265625, "grad_norm": 0.6939353942871094, "learning_rate": 7.808402738346527e-05, "loss": 0.015481479465961456, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.0156, "step": 186, "tokens/total": 6051616, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 85296} +{"epoch": 0.73046875, "grad_norm": 0.5183178186416626, "learning_rate": 7.783436264872382e-05, "loss": 0.013425535522401333, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.54, "memory/max_allocated (GiB)": 64.54, "ppl": 1.01352, "step": 187, "tokens/total": 6084304, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 85741} +{"epoch": 0.734375, "grad_norm": 0.3700525164604187, "learning_rate": 7.758374768294647e-05, "loss": 0.006675977259874344, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.0067, "step": 188, "tokens/total": 6117896, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 86176} +{"epoch": 0.73828125, "grad_norm": 0.27529940009117126, "learning_rate": 7.733219291524489e-05, "loss": 0.005108041688799858, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00512, "step": 189, "tokens/total": 6152296, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 86649} +{"epoch": 0.7421875, "grad_norm": 0.4987492561340332, "learning_rate": 7.707970881383977e-05, "loss": 0.018522903323173523, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.0187, "step": 190, "tokens/total": 6186056, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 87084} +{"epoch": 0.74609375, "grad_norm": 0.3436739444732666, "learning_rate": 7.682630588562518e-05, "loss": 0.009877990931272507, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00993, "step": 191, "tokens/total": 6219160, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 87499} +{"epoch": 0.75, "grad_norm": 0.1960086077451706, "learning_rate": 7.657199467573129e-05, "loss": 0.0023719340097159147, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00237, "step": 192, "tokens/total": 6254840, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 87987} +{"epoch": 0.75390625, "grad_norm": 0.3241189420223236, "learning_rate": 7.631678576708561e-05, "loss": 0.004649649374186993, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00466, "step": 193, "tokens/total": 6289656, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 88470} +{"epoch": 0.7578125, "grad_norm": 0.1977485865354538, "learning_rate": 7.606068977997255e-05, "loss": 0.0029363350477069616, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00294, "step": 194, "tokens/total": 6321776, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 88923} +{"epoch": 0.76171875, "grad_norm": 0.713251531124115, "learning_rate": 7.580371737159148e-05, "loss": 0.013285533525049686, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.01337, "step": 195, "tokens/total": 6352824, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 89338} +{"epoch": 0.765625, "grad_norm": 0.199573814868927, "learning_rate": 7.554587923561324e-05, "loss": 0.00676954910159111, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00679, "step": 196, "tokens/total": 6388496, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 89812} +{"epoch": 0.76953125, "grad_norm": 0.6638284921646118, "learning_rate": 7.528718610173511e-05, "loss": 0.010103792883455753, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.01016, "step": 197, "tokens/total": 6420600, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 90252} +{"epoch": 0.7734375, "grad_norm": 0.37103205919265747, "learning_rate": 7.502764873523431e-05, "loss": 0.0058985925279557705, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00592, "step": 198, "tokens/total": 6454616, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 90718} +{"epoch": 0.77734375, "grad_norm": 0.5865607261657715, "learning_rate": 7.476727793652011e-05, "loss": 0.01105555146932602, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.01112, "step": 199, "tokens/total": 6485680, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 91194} +{"epoch": 0.78125, "grad_norm": 0.8302373290061951, "learning_rate": 7.450608454068415e-05, "loss": 0.01140798069536686, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.01147, "step": 200, "tokens/total": 6518512, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 91649} +{"epoch": 0.78515625, "grad_norm": 0.2837420403957367, "learning_rate": 7.424407941704987e-05, "loss": 0.001186978304758668, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.00119, "step": 201, "tokens/total": 6548728, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 92078} +{"epoch": 0.7890625, "grad_norm": 1.3439960479736328, "learning_rate": 7.398127346871986e-05, "loss": 0.02041688561439514, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.02063, "step": 202, "tokens/total": 6582192, "tokens/train_per_sec_per_gpu": 7.24, "tokens/trainable": 92536} +{"epoch": 0.79296875, "grad_norm": 0.5211635231971741, "learning_rate": 7.371767763212238e-05, "loss": 0.013585295528173447, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.01368, "step": 203, "tokens/total": 6617392, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 93032} +{"epoch": 0.796875, "grad_norm": 0.12850604951381683, "learning_rate": 7.345330287655617e-05, "loss": 0.001487487810663879, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00149, "step": 204, "tokens/total": 6648904, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 93504} +{"epoch": 0.80078125, "grad_norm": 0.2296265810728073, "learning_rate": 7.31881602037339e-05, "loss": 0.0050790878012776375, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00509, "step": 205, "tokens/total": 6679832, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 93925} +{"epoch": 0.8046875, "grad_norm": 0.5772904753684998, "learning_rate": 7.29222606473245e-05, "loss": 0.0071587758138775826, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00718, "step": 206, "tokens/total": 6710784, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 94365} +{"epoch": 0.80859375, "grad_norm": 0.4770526587963104, "learning_rate": 7.265561527249383e-05, "loss": 0.007906449027359486, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00794, "step": 207, "tokens/total": 6744848, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 94845} +{"epoch": 0.8125, "grad_norm": 0.4107087254524231, "learning_rate": 7.238823517544436e-05, "loss": 0.008234163746237755, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00827, "step": 208, "tokens/total": 6777600, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 95285} +{"epoch": 0.81640625, "grad_norm": 0.9653295278549194, "learning_rate": 7.212013148295333e-05, "loss": 0.009926512837409973, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00998, "step": 209, "tokens/total": 6811024, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 95758} +{"epoch": 0.8203125, "grad_norm": 0.197322815656662, "learning_rate": 7.185131535190975e-05, "loss": 0.003990075550973415, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.15, "memory/max_allocated (GiB)": 66.15, "ppl": 1.004, "step": 210, "tokens/total": 6845376, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 96192} +{"epoch": 0.82421875, "grad_norm": 0.6267563700675964, "learning_rate": 7.158179796885005e-05, "loss": 0.027865130454301834, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.02826, "step": 211, "tokens/total": 6880776, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 96641} +{"epoch": 0.828125, "grad_norm": 0.9538257718086243, "learning_rate": 7.131159054949273e-05, "loss": 0.02814575843513012, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.02855, "step": 212, "tokens/total": 6916160, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 97160} +{"epoch": 0.83203125, "grad_norm": 0.3003867268562317, "learning_rate": 7.104070433827139e-05, "loss": 0.0027172660920768976, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.54, "memory/max_allocated (GiB)": 63.54, "ppl": 1.00272, "step": 213, "tokens/total": 6945320, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 97611} +{"epoch": 0.8359375, "grad_norm": 0.5080604553222656, "learning_rate": 7.076915060786705e-05, "loss": 0.007997800596058369, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00803, "step": 214, "tokens/total": 6977856, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 98111} +{"epoch": 0.83984375, "grad_norm": 0.6628919243812561, "learning_rate": 7.049694065873882e-05, "loss": 0.007551256101578474, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.00758, "step": 215, "tokens/total": 7011096, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 98575} +{"epoch": 0.84375, "grad_norm": 0.1428976207971573, "learning_rate": 7.022408581865382e-05, "loss": 0.00226710457354784, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.93, "memory/max_allocated (GiB)": 63.93, "ppl": 1.00227, "step": 216, "tokens/total": 7041408, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 98991} +{"epoch": 0.84765625, "grad_norm": 0.5045188665390015, "learning_rate": 6.99505974422157e-05, "loss": 0.00368921784684062, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.6, "memory/max_allocated (GiB)": 63.6, "ppl": 1.0037, "step": 217, "tokens/total": 7071560, "tokens/train_per_sec_per_gpu": 11.32, "tokens/trainable": 99454} +{"epoch": 0.8515625, "grad_norm": 0.019153473898768425, "learning_rate": 6.967648691039213e-05, "loss": 0.00044234393863007426, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00044, "step": 218, "tokens/total": 7104696, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 99925} +{"epoch": 0.85546875, "grad_norm": 0.17338955402374268, "learning_rate": 6.940176563004123e-05, "loss": 0.0019081196514889598, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00191, "step": 219, "tokens/total": 7138504, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 100371} +{"epoch": 0.859375, "grad_norm": 0.462173730134964, "learning_rate": 6.912644503343682e-05, "loss": 0.006547209806740284, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00657, "step": 220, "tokens/total": 7170992, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 100855} +{"epoch": 0.86328125, "grad_norm": 0.7114114165306091, "learning_rate": 6.885053657779273e-05, "loss": 0.0017227944917976856, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00172, "step": 221, "tokens/total": 7201176, "tokens/train_per_sec_per_gpu": 6.76, "tokens/trainable": 101253} +{"epoch": 0.8671875, "grad_norm": 0.2588549852371216, "learning_rate": 6.857405174478604e-05, "loss": 0.0023190462961792946, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00232, "step": 222, "tokens/total": 7233528, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 101706} +{"epoch": 0.87109375, "grad_norm": 0.40632525086402893, "learning_rate": 6.82970020400792e-05, "loss": 0.00977163203060627, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00982, "step": 223, "tokens/total": 7267672, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 102158} +{"epoch": 0.875, "grad_norm": 1.3911762237548828, "learning_rate": 6.801939899284132e-05, "loss": 0.014443440362811089, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 62.86, "memory/max_allocated (GiB)": 62.86, "ppl": 1.01455, "step": 224, "tokens/total": 7295592, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 102580} +{"epoch": 0.87890625, "grad_norm": 1.8606321811676025, "learning_rate": 6.774125415526827e-05, "loss": 0.00793613400310278, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00797, "step": 225, "tokens/total": 7325816, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 103063} +{"epoch": 0.8828125, "grad_norm": 0.42138874530792236, "learning_rate": 6.746257910210214e-05, "loss": 0.0037656158674508333, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00377, "step": 226, "tokens/total": 7360888, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 103510} +{"epoch": 0.88671875, "grad_norm": 0.0355207733809948, "learning_rate": 6.718338543014937e-05, "loss": 0.0003867318737320602, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00039, "step": 227, "tokens/total": 7390416, "tokens/train_per_sec_per_gpu": 10.1, "tokens/trainable": 103980} +{"epoch": 0.890625, "grad_norm": 1.915993571281433, "learning_rate": 6.69036847577983e-05, "loss": 0.0033766382839530706, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00338, "step": 228, "tokens/total": 7420480, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 104413} +{"epoch": 0.89453125, "grad_norm": 0.5045561194419861, "learning_rate": 6.662348872453553e-05, "loss": 0.01144988089799881, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.01152, "step": 229, "tokens/total": 7448992, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 104845} +{"epoch": 0.8984375, "grad_norm": 0.2933925986289978, "learning_rate": 6.63428089904618e-05, "loss": 0.004899743013083935, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00491, "step": 230, "tokens/total": 7481744, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 105289} +{"epoch": 0.90234375, "grad_norm": 0.4187624752521515, "learning_rate": 6.60616572358065e-05, "loss": 0.01088013593107462, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01094, "step": 231, "tokens/total": 7516704, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 105722} +{"epoch": 0.90625, "grad_norm": 0.7703221440315247, "learning_rate": 6.578004516044172e-05, "loss": 0.007178358733654022, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.19, "memory/max_allocated (GiB)": 64.19, "ppl": 1.0072, "step": 232, "tokens/total": 7548592, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 106213} +{"epoch": 0.91015625, "grad_norm": 0.6207764744758606, "learning_rate": 6.549798448339548e-05, "loss": 0.008574034087359905, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00861, "step": 233, "tokens/total": 7579288, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 106643} +{"epoch": 0.9140625, "grad_norm": 0.5156052708625793, "learning_rate": 6.521548694236384e-05, "loss": 0.012023590505123138, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.0121, "step": 234, "tokens/total": 7611672, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 107136} +{"epoch": 0.91796875, "grad_norm": 0.4275663495063782, "learning_rate": 6.493256429322259e-05, "loss": 0.006811156868934631, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00683, "step": 235, "tokens/total": 7646288, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 107588} +{"epoch": 0.921875, "grad_norm": 0.06834395229816437, "learning_rate": 6.464922830953799e-05, "loss": 0.0012817330425605178, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00128, "step": 236, "tokens/total": 7680136, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 108087} +{"epoch": 0.92578125, "grad_norm": 1.6735095977783203, "learning_rate": 6.436549078207688e-05, "loss": 0.007571800611913204, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.61, "memory/max_allocated (GiB)": 64.61, "ppl": 1.0076, "step": 237, "tokens/total": 7713072, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 108533} +{"epoch": 0.9296875, "grad_norm": 0.03791302442550659, "learning_rate": 6.408136351831592e-05, "loss": 0.0008591668447479606, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00086, "step": 238, "tokens/total": 7747856, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 108965} +{"epoch": 0.93359375, "grad_norm": 0.1498965322971344, "learning_rate": 6.379685834195036e-05, "loss": 0.002474042819812894, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00248, "step": 239, "tokens/total": 7780136, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 109416} +{"epoch": 0.9375, "grad_norm": 0.38084328174591064, "learning_rate": 6.351198709240186e-05, "loss": 0.0033182031475007534, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.78, "memory/max_allocated (GiB)": 65.78, "ppl": 1.00332, "step": 240, "tokens/total": 7816344, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 109864} +{"epoch": 0.94140625, "grad_norm": 0.22994334995746613, "learning_rate": 6.32267616243259e-05, "loss": 0.003722946858033538, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00373, "step": 241, "tokens/total": 7850408, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 110345} +{"epoch": 0.9453125, "grad_norm": 0.0627540647983551, "learning_rate": 6.294119380711849e-05, "loss": 0.0011560183484107256, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00116, "step": 242, "tokens/total": 7881528, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 110794} +{"epoch": 0.94921875, "grad_norm": 0.10045754909515381, "learning_rate": 6.265529552442209e-05, "loss": 0.0008201185264624655, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00082, "step": 243, "tokens/total": 7915712, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 111250} +{"epoch": 0.953125, "grad_norm": 0.24707859754562378, "learning_rate": 6.236907867363127e-05, "loss": 0.002922828309237957, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.61, "memory/max_allocated (GiB)": 64.61, "ppl": 1.00293, "step": 244, "tokens/total": 7947408, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 111693} +{"epoch": 0.95703125, "grad_norm": 0.6027008295059204, "learning_rate": 6.208255516539749e-05, "loss": 0.008013129234313965, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.26, "memory/max_allocated (GiB)": 65.26, "ppl": 1.00805, "step": 245, "tokens/total": 7979600, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 112158} +{"epoch": 0.9609375, "grad_norm": 0.296305775642395, "learning_rate": 6.179573692313344e-05, "loss": 0.003842368256300688, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00385, "step": 246, "tokens/total": 8010848, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 112601} +{"epoch": 0.96484375, "grad_norm": 0.09604158997535706, "learning_rate": 6.150863588251694e-05, "loss": 0.0009123932104557753, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.00091, "step": 247, "tokens/total": 8040632, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 113016} +{"epoch": 0.96875, "grad_norm": 0.1096530333161354, "learning_rate": 6.122126399099419e-05, "loss": 0.0006083159241825342, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00061, "step": 248, "tokens/total": 8072464, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 113476} +{"epoch": 0.97265625, "grad_norm": 0.0724371150135994, "learning_rate": 6.0933633207282615e-05, "loss": 0.0006577699678018689, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.96, "memory/max_allocated (GiB)": 64.96, "ppl": 1.00066, "step": 249, "tokens/total": 8106568, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 113941} +{"epoch": 0.9765625, "grad_norm": 0.4160044491291046, "learning_rate": 6.064575550087316e-05, "loss": 0.004686020314693451, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.0047, "step": 250, "tokens/total": 8138496, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 114434} +{"epoch": 0.98046875, "grad_norm": 0.008254989981651306, "learning_rate": 6.0357642851532245e-05, "loss": 9.340287215309218e-05, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00009, "step": 251, "tokens/total": 8168080, "tokens/train_per_sec_per_gpu": 10.2, "tokens/trainable": 114898} +{"epoch": 0.984375, "grad_norm": 0.0724160298705101, "learning_rate": 6.0069307248803294e-05, "loss": 0.0002707880630623549, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00027, "step": 252, "tokens/total": 8200888, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 115331} +{"epoch": 0.98828125, "grad_norm": 0.8523172736167908, "learning_rate": 5.9780760691507635e-05, "loss": 0.0171223022043705, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.01727, "step": 253, "tokens/total": 8230704, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 115732} +{"epoch": 0.9921875, "grad_norm": 0.6460132598876953, "learning_rate": 5.9492015187245334e-05, "loss": 0.00971921905875206, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00977, "step": 254, "tokens/total": 8266560, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 116221} +{"epoch": 0.99609375, "grad_norm": 0.5586274862289429, "learning_rate": 5.920308275189541e-05, "loss": 0.0122835086658597, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.19, "memory/max_allocated (GiB)": 65.19, "ppl": 1.01236, "step": 255, "tokens/total": 8300184, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 116684} +{"epoch": 1.0, "grad_norm": 0.5727865099906921, "learning_rate": 5.8913975409115874e-05, "loss": 0.00702905235812068, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00705, "step": 256, "tokens/total": 8333624, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 117126} +{"epoch": 1.00390625, "grad_norm": 1.0057581663131714, "learning_rate": 5.8624705189843395e-05, "loss": 0.0061002252623438835, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00612, "step": 257, "tokens/total": 8364976, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 117592} +{"epoch": 1.0078125, "grad_norm": 0.0708262175321579, "learning_rate": 5.833528413179249e-05, "loss": 0.0007922882214188576, "memory/device_reserved (GiB)": 66.62, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00079, "step": 258, "tokens/total": 8399568, "tokens/train_per_sec_per_gpu": 6.94, "tokens/trainable": 118030} +{"epoch": 1.01171875, "grad_norm": 0.25528377294540405, "learning_rate": 5.80457242789548e-05, "loss": 0.012595786713063717, "memory/device_reserved (GiB)": 66.62, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.01268, "step": 259, "tokens/total": 8430616, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 118470} +{"epoch": 1.015625, "grad_norm": 0.467791348695755, "learning_rate": 5.77560376810977e-05, "loss": 0.009190365672111511, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00923, "step": 260, "tokens/total": 8462696, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 118935} +{"epoch": 1.01953125, "grad_norm": 0.14819562435150146, "learning_rate": 5.7466236393263005e-05, "loss": 0.001615309389308095, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00162, "step": 261, "tokens/total": 8496152, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 119381} +{"epoch": 1.0234375, "grad_norm": 0.31563133001327515, "learning_rate": 5.717633247526522e-05, "loss": 0.005431316327303648, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00545, "step": 262, "tokens/total": 8532912, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 119849} +{"epoch": 1.02734375, "grad_norm": 0.06369991600513458, "learning_rate": 5.688633799118971e-05, "loss": 0.0005267371307127178, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.00053, "step": 263, "tokens/total": 8565376, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 120319} +{"epoch": 1.03125, "grad_norm": 0.19077858328819275, "learning_rate": 5.659626500889066e-05, "loss": 0.0026257664430886507, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00263, "step": 264, "tokens/total": 8597640, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 120804} +{"epoch": 1.03515625, "grad_norm": 0.31643977761268616, "learning_rate": 5.6306125599488905e-05, "loss": 0.003402209375053644, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.00341, "step": 265, "tokens/total": 8627312, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 121233} +{"epoch": 1.0390625, "grad_norm": 0.41560012102127075, "learning_rate": 5.601593183686955e-05, "loss": 0.005564745049923658, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.87, "memory/max_allocated (GiB)": 64.87, "ppl": 1.00558, "step": 266, "tokens/total": 8659208, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 121671} +{"epoch": 1.04296875, "grad_norm": 0.0895494818687439, "learning_rate": 5.572569579717961e-05, "loss": 0.0008485516300424933, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.00085, "step": 267, "tokens/total": 8693088, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 122141} +{"epoch": 1.046875, "grad_norm": 0.11094445735216141, "learning_rate": 5.543542955832538e-05, "loss": 0.0011773909209296107, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00118, "step": 268, "tokens/total": 8725576, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 122556} +{"epoch": 1.05078125, "grad_norm": 0.648537278175354, "learning_rate": 5.514514519946986e-05, "loss": 0.019414862617850304, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.0196, "step": 269, "tokens/total": 8755192, "tokens/train_per_sec_per_gpu": 10.3, "tokens/trainable": 122988} +{"epoch": 1.0546875, "grad_norm": 0.018498605117201805, "learning_rate": 5.485485480053015e-05, "loss": 0.00044270209036767483, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00044, "step": 270, "tokens/total": 8787704, "tokens/train_per_sec_per_gpu": 10.5, "tokens/trainable": 123478} +{"epoch": 1.05859375, "grad_norm": 0.3001265525817871, "learning_rate": 5.4564570441674645e-05, "loss": 0.003864526515826583, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00387, "step": 271, "tokens/total": 8819768, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 123932} +{"epoch": 1.0625, "grad_norm": 0.38471272587776184, "learning_rate": 5.42743042028204e-05, "loss": 0.007919508032500744, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00795, "step": 272, "tokens/total": 8853352, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 124428} +{"epoch": 1.06640625, "grad_norm": 0.9517339468002319, "learning_rate": 5.3984068163130464e-05, "loss": 0.007675399072468281, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.0077, "step": 273, "tokens/total": 8886840, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 124866} +{"epoch": 1.0703125, "grad_norm": 0.23769725859165192, "learning_rate": 5.369387440051111e-05, "loss": 0.004702525679022074, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00471, "step": 274, "tokens/total": 8920808, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 125324} +{"epoch": 1.07421875, "grad_norm": 0.28322118520736694, "learning_rate": 5.340373499110935e-05, "loss": 0.005348049104213715, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00536, "step": 275, "tokens/total": 8955880, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 125813} +{"epoch": 1.078125, "grad_norm": 0.03942309692502022, "learning_rate": 5.3113662008810304e-05, "loss": 0.000737656548153609, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.00074, "step": 276, "tokens/total": 8989520, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 126252} +{"epoch": 1.08203125, "grad_norm": 0.03845078498125076, "learning_rate": 5.282366752473479e-05, "loss": 0.0008660503663122654, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00087, "step": 277, "tokens/total": 9024680, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 126696} +{"epoch": 1.0859375, "grad_norm": 0.27021944522857666, "learning_rate": 5.2533763606737005e-05, "loss": 0.006965824402868748, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00699, "step": 278, "tokens/total": 9056208, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 127173} +{"epoch": 1.08984375, "grad_norm": 0.35260140895843506, "learning_rate": 5.224396231890232e-05, "loss": 0.006275273859500885, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.0063, "step": 279, "tokens/total": 9086376, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 127637} +{"epoch": 1.09375, "grad_norm": 0.1960863173007965, "learning_rate": 5.195427572104522e-05, "loss": 0.0028780989814549685, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00288, "step": 280, "tokens/total": 9118944, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 128112} +{"epoch": 1.09765625, "grad_norm": 0.1366867572069168, "learning_rate": 5.166471586820751e-05, "loss": 0.0013287363108247519, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00133, "step": 281, "tokens/total": 9152920, "tokens/train_per_sec_per_gpu": 7.0, "tokens/trainable": 128584} +{"epoch": 1.1015625, "grad_norm": 0.20655180513858795, "learning_rate": 5.1375294810156615e-05, "loss": 0.0015210759593173862, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00152, "step": 282, "tokens/total": 9185728, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 129051} +{"epoch": 1.10546875, "grad_norm": 0.08630593121051788, "learning_rate": 5.1086024590884144e-05, "loss": 0.0011987444013357162, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.0012, "step": 283, "tokens/total": 9217984, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 129494} +{"epoch": 1.109375, "grad_norm": 0.2462371289730072, "learning_rate": 5.079691724810461e-05, "loss": 0.005576981231570244, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00559, "step": 284, "tokens/total": 9253160, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 129983} +{"epoch": 1.11328125, "grad_norm": 0.05571060627698898, "learning_rate": 5.0507984812754684e-05, "loss": 0.0009647426777519286, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00097, "step": 285, "tokens/total": 9286136, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 130405} +{"epoch": 1.1171875, "grad_norm": 0.34204742312431335, "learning_rate": 5.021923930849237e-05, "loss": 0.005387254059314728, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.0054, "step": 286, "tokens/total": 9318360, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 130829} +{"epoch": 1.12109375, "grad_norm": 0.4046356976032257, "learning_rate": 4.99306927511967e-05, "loss": 0.003227128414437175, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00323, "step": 287, "tokens/total": 9350288, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 131287} +{"epoch": 1.125, "grad_norm": 0.42977291345596313, "learning_rate": 4.964235714846775e-05, "loss": 0.010567554272711277, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.01062, "step": 288, "tokens/total": 9382104, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 131712} +{"epoch": 1.12890625, "grad_norm": 0.03184682875871658, "learning_rate": 4.9354244499126866e-05, "loss": 0.00031776601099409163, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00032, "step": 289, "tokens/total": 9414392, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 132168} +{"epoch": 1.1328125, "grad_norm": 0.12416297942399979, "learning_rate": 4.90663667927174e-05, "loss": 0.0017190339276567101, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00172, "step": 290, "tokens/total": 9447112, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 132626} +{"epoch": 1.13671875, "grad_norm": 0.025938605889678, "learning_rate": 4.877873600900581e-05, "loss": 0.0002866282011382282, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00029, "step": 291, "tokens/total": 9483400, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 133119} +{"epoch": 1.140625, "grad_norm": 0.007283744867891073, "learning_rate": 4.849136411748306e-05, "loss": 0.00012209809210617095, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.48, "memory/max_allocated (GiB)": 64.48, "ppl": 1.00012, "step": 292, "tokens/total": 9513896, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 133550} +{"epoch": 1.14453125, "grad_norm": 0.0441540852189064, "learning_rate": 4.8204263076866574e-05, "loss": 0.00025281048147007823, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00025, "step": 293, "tokens/total": 9550184, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 133992} +{"epoch": 1.1484375, "grad_norm": 0.009887062013149261, "learning_rate": 4.791744483460251e-05, "loss": 0.00017821112123783678, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00018, "step": 294, "tokens/total": 9583920, "tokens/train_per_sec_per_gpu": 10.06, "tokens/trainable": 134467} +{"epoch": 1.15234375, "grad_norm": 0.2490794062614441, "learning_rate": 4.7630921326368736e-05, "loss": 0.0019156079506501555, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00192, "step": 295, "tokens/total": 9617288, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 134895} +{"epoch": 1.15625, "grad_norm": 0.04324949160218239, "learning_rate": 4.7344704475577916e-05, "loss": 0.00026097235968336463, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 62.94, "memory/max_allocated (GiB)": 62.94, "ppl": 1.00026, "step": 296, "tokens/total": 9645600, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 135321} +{"epoch": 1.16015625, "grad_norm": 0.031133826822042465, "learning_rate": 4.705880619288153e-05, "loss": 0.0003535154974088073, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00035, "step": 297, "tokens/total": 9679456, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 135765} +{"epoch": 1.1640625, "grad_norm": 0.013924206607043743, "learning_rate": 4.677323837567412e-05, "loss": 0.000203195697395131, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.0002, "step": 298, "tokens/total": 9711696, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 136196} +{"epoch": 1.16796875, "grad_norm": 0.0809149369597435, "learning_rate": 4.6488012907598146e-05, "loss": 0.0009880918078124523, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00099, "step": 299, "tokens/total": 9745792, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 136649} +{"epoch": 1.171875, "grad_norm": 0.7845627665519714, "learning_rate": 4.620314165804964e-05, "loss": 0.007949676364660263, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00798, "step": 300, "tokens/total": 9780648, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 137128} +{"epoch": 1.17578125, "grad_norm": 1.709397554397583, "learning_rate": 4.591863648168407e-05, "loss": 0.011371376924216747, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.01144, "step": 301, "tokens/total": 9813032, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 137585} +{"epoch": 1.1796875, "grad_norm": 0.011140134185552597, "learning_rate": 4.5634509217923135e-05, "loss": 0.00012671187869273126, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00013, "step": 302, "tokens/total": 9845200, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 138054} +{"epoch": 1.18359375, "grad_norm": 0.07272807508707047, "learning_rate": 4.535077169046201e-05, "loss": 0.0007118968642316759, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.29, "memory/max_allocated (GiB)": 65.29, "ppl": 1.00071, "step": 303, "tokens/total": 9878616, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 138559} +{"epoch": 1.1875, "grad_norm": 0.7119264602661133, "learning_rate": 4.506743570677743e-05, "loss": 0.007932665757834911, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00796, "step": 304, "tokens/total": 9910776, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 139050} +{"epoch": 1.19140625, "grad_norm": 0.08967869728803635, "learning_rate": 4.478451305763618e-05, "loss": 0.0007771769887767732, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00078, "step": 305, "tokens/total": 9945480, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 139525} +{"epoch": 1.1953125, "grad_norm": 0.08402936160564423, "learning_rate": 4.450201551660454e-05, "loss": 0.0005135919782333076, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.00051, "step": 306, "tokens/total": 9975992, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 139969} +{"epoch": 1.19921875, "grad_norm": 0.036437734961509705, "learning_rate": 4.4219954839558276e-05, "loss": 0.00028359395219013095, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.00028, "step": 307, "tokens/total": 10006440, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 140443} +{"epoch": 1.203125, "grad_norm": 0.020253796130418777, "learning_rate": 4.393834276419352e-05, "loss": 0.00023335831065196544, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00023, "step": 308, "tokens/total": 10042928, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 140926} +{"epoch": 1.20703125, "grad_norm": 0.3850789964199066, "learning_rate": 4.36571910095382e-05, "loss": 0.0053000012412667274, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00531, "step": 309, "tokens/total": 10075184, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 141344} +{"epoch": 1.2109375, "grad_norm": 0.1670452207326889, "learning_rate": 4.337651127546448e-05, "loss": 0.0013116298941895366, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00131, "step": 310, "tokens/total": 10106656, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 141753} +{"epoch": 1.21484375, "grad_norm": 0.39783990383148193, "learning_rate": 4.3096315242201736e-05, "loss": 0.00285080773755908, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.82, "memory/max_allocated (GiB)": 63.82, "ppl": 1.00285, "step": 311, "tokens/total": 10138536, "tokens/train_per_sec_per_gpu": 10.05, "tokens/trainable": 142219} +{"epoch": 1.21875, "grad_norm": 0.3068773150444031, "learning_rate": 4.2816614569850635e-05, "loss": 0.0041300286538898945, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.00414, "step": 312, "tokens/total": 10165768, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 142617} +{"epoch": 1.22265625, "grad_norm": 0.07855836302042007, "learning_rate": 4.2537420897897864e-05, "loss": 0.0005437790532596409, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00054, "step": 313, "tokens/total": 10201624, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 143103} +{"epoch": 1.2265625, "grad_norm": 0.6096007823944092, "learning_rate": 4.225874584473174e-05, "loss": 0.012736774049699306, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.01282, "step": 314, "tokens/total": 10233304, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 143535} +{"epoch": 1.23046875, "grad_norm": 0.09146911650896072, "learning_rate": 4.19806010071587e-05, "loss": 0.0006218075286597013, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00062, "step": 315, "tokens/total": 10267904, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 144009} +{"epoch": 1.234375, "grad_norm": 0.7725475430488586, "learning_rate": 4.170299795992081e-05, "loss": 0.007732695899903774, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00776, "step": 316, "tokens/total": 10301280, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 144460} +{"epoch": 1.23828125, "grad_norm": 0.09256047010421753, "learning_rate": 4.142594825521398e-05, "loss": 0.0010821683099493384, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00108, "step": 317, "tokens/total": 10337072, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 144936} +{"epoch": 1.2421875, "grad_norm": 0.011213993653655052, "learning_rate": 4.114946342220728e-05, "loss": 0.00014242382894735783, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.69, "memory/max_allocated (GiB)": 64.69, "ppl": 1.00014, "step": 318, "tokens/total": 10368688, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 145375} +{"epoch": 1.24609375, "grad_norm": 0.33247488737106323, "learning_rate": 4.087355496656321e-05, "loss": 0.00283968192525208, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00284, "step": 319, "tokens/total": 10400752, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 145849} +{"epoch": 1.25, "grad_norm": 0.10205478966236115, "learning_rate": 4.05982343699588e-05, "loss": 0.0013698508264496922, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00137, "step": 320, "tokens/total": 10432112, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 146318} +{"epoch": 1.25390625, "grad_norm": 0.05026552081108093, "learning_rate": 4.0323513089607876e-05, "loss": 0.0004976389463990927, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.0005, "step": 321, "tokens/total": 10465472, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 146821} +{"epoch": 1.2578125, "grad_norm": 0.03785328194499016, "learning_rate": 4.004940255778431e-05, "loss": 0.00023647370107937604, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00024, "step": 322, "tokens/total": 10496336, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 147285} +{"epoch": 1.26171875, "grad_norm": 0.013677257113158703, "learning_rate": 3.977591418134619e-05, "loss": 0.00019079737830907106, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00019, "step": 323, "tokens/total": 10525864, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 147729} +{"epoch": 1.265625, "grad_norm": 0.32207179069519043, "learning_rate": 3.95030593412612e-05, "loss": 0.0014726142399013042, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00147, "step": 324, "tokens/total": 10559680, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 148169} +{"epoch": 1.26953125, "grad_norm": 0.22804619371891022, "learning_rate": 3.923084939213296e-05, "loss": 0.001817650510929525, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00182, "step": 325, "tokens/total": 10595400, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 148630} +{"epoch": 1.2734375, "grad_norm": 0.03522114083170891, "learning_rate": 3.895929566172861e-05, "loss": 0.00043368610204197466, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00043, "step": 326, "tokens/total": 10626680, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 149063} +{"epoch": 1.27734375, "grad_norm": 0.19942647218704224, "learning_rate": 3.868840945050728e-05, "loss": 0.0011213056277483702, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00112, "step": 327, "tokens/total": 10661784, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 149549} +{"epoch": 1.28125, "grad_norm": 0.3419588506221771, "learning_rate": 3.841820203114995e-05, "loss": 0.01112514641135931, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.31, "memory/max_allocated (GiB)": 65.31, "ppl": 1.01119, "step": 328, "tokens/total": 10696184, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 150015} +{"epoch": 1.28515625, "grad_norm": 0.04273435100913048, "learning_rate": 3.814868464809027e-05, "loss": 0.0002543386653997004, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00025, "step": 329, "tokens/total": 10729376, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 150514} +{"epoch": 1.2890625, "grad_norm": 0.003291552420705557, "learning_rate": 3.787986851704667e-05, "loss": 4.872860881732777e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00005, "step": 330, "tokens/total": 10764824, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 151021} +{"epoch": 1.29296875, "grad_norm": 0.38219940662384033, "learning_rate": 3.7611764824555654e-05, "loss": 0.003262273268774152, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00327, "step": 331, "tokens/total": 10797072, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 151486} +{"epoch": 1.296875, "grad_norm": 0.3060857951641083, "learning_rate": 3.734438472750619e-05, "loss": 0.0032952202018350363, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.0033, "step": 332, "tokens/total": 10829376, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 151956} +{"epoch": 1.30078125, "grad_norm": 0.034680236130952835, "learning_rate": 3.707773935267552e-05, "loss": 0.0004360602470114827, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00044, "step": 333, "tokens/total": 10863512, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 152442} +{"epoch": 1.3046875, "grad_norm": 0.02024129405617714, "learning_rate": 3.68118397962661e-05, "loss": 0.0003442860906943679, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00034, "step": 334, "tokens/total": 10894432, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 152890} +{"epoch": 1.30859375, "grad_norm": 0.6909961104393005, "learning_rate": 3.654669712344384e-05, "loss": 0.016671115532517433, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.34, "memory/max_allocated (GiB)": 63.34, "ppl": 1.01681, "step": 335, "tokens/total": 10924848, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 153368} +{"epoch": 1.3125, "grad_norm": 0.014136377722024918, "learning_rate": 3.628232236787763e-05, "loss": 0.000178085858351551, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00018, "step": 336, "tokens/total": 10959024, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 153809} +{"epoch": 1.31640625, "grad_norm": 1.1360764503479004, "learning_rate": 3.6018726531280144e-05, "loss": 0.012047764845192432, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.01212, "step": 337, "tokens/total": 10991088, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 154258} +{"epoch": 1.3203125, "grad_norm": 0.08495453000068665, "learning_rate": 3.575592058295017e-05, "loss": 0.001393817481584847, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00139, "step": 338, "tokens/total": 11023904, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 154704} +{"epoch": 1.32421875, "grad_norm": 0.0211932472884655, "learning_rate": 3.549391545931585e-05, "loss": 0.0003830472123809159, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00038, "step": 339, "tokens/total": 11057680, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 155157} +{"epoch": 1.328125, "grad_norm": 0.17644751071929932, "learning_rate": 3.5232722063479914e-05, "loss": 0.001575552043505013, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00158, "step": 340, "tokens/total": 11089800, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 155582} +{"epoch": 1.33203125, "grad_norm": 0.013502120971679688, "learning_rate": 3.49723512647657e-05, "loss": 0.00021908619964960963, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00022, "step": 341, "tokens/total": 11117960, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 156047} +{"epoch": 1.3359375, "grad_norm": 0.15003333985805511, "learning_rate": 3.471281389826491e-05, "loss": 0.0022201614920049906, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00222, "step": 342, "tokens/total": 11151888, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 156500} +{"epoch": 1.33984375, "grad_norm": 0.06293938308954239, "learning_rate": 3.4454120764386764e-05, "loss": 0.0007534095784649253, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.22, "memory/max_allocated (GiB)": 65.22, "ppl": 1.00075, "step": 343, "tokens/total": 11184568, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 156931} +{"epoch": 1.34375, "grad_norm": 0.41624921560287476, "learning_rate": 3.4196282628408526e-05, "loss": 0.01729961298406124, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.01745, "step": 344, "tokens/total": 11216400, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 157385} +{"epoch": 1.34765625, "grad_norm": 0.022796381264925003, "learning_rate": 3.3939310220027456e-05, "loss": 0.0003897428687196225, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00039, "step": 345, "tokens/total": 11250768, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 157817} +{"epoch": 1.3515625, "grad_norm": 0.36852625012397766, "learning_rate": 3.3683214232914404e-05, "loss": 0.008107427507638931, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00814, "step": 346, "tokens/total": 11281808, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 158273} +{"epoch": 1.35546875, "grad_norm": 0.06058458238840103, "learning_rate": 3.342800532426873e-05, "loss": 0.001452813041396439, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00145, "step": 347, "tokens/total": 11313968, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 158728} +{"epoch": 1.359375, "grad_norm": 0.00767044723033905, "learning_rate": 3.317369411437484e-05, "loss": 0.00012137380690546706, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00012, "step": 348, "tokens/total": 11347288, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 159168} +{"epoch": 1.36328125, "grad_norm": 0.3197716474533081, "learning_rate": 3.292029118616024e-05, "loss": 0.004521036986261606, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00453, "step": 349, "tokens/total": 11377128, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 159592} +{"epoch": 1.3671875, "grad_norm": 0.12458527833223343, "learning_rate": 3.266780708475511e-05, "loss": 0.0013966106344014406, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.0014, "step": 350, "tokens/total": 11409488, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 160093} +{"epoch": 1.37109375, "grad_norm": 0.1060534343123436, "learning_rate": 3.241625231705354e-05, "loss": 0.0006070779054425657, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00061, "step": 351, "tokens/total": 11442040, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 160534} +{"epoch": 1.375, "grad_norm": 0.018948961049318314, "learning_rate": 3.216563735127618e-05, "loss": 0.00027427406166680157, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00027, "step": 352, "tokens/total": 11478312, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 161009} +{"epoch": 1.37890625, "grad_norm": 1.392184853553772, "learning_rate": 3.191597261653475e-05, "loss": 0.008588782511651516, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.93, "memory/max_allocated (GiB)": 63.93, "ppl": 1.00863, "step": 353, "tokens/total": 11509008, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 161450} +{"epoch": 1.3828125, "grad_norm": 0.7611889839172363, "learning_rate": 3.166726850239794e-05, "loss": 0.025555303320288658, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.02588, "step": 354, "tokens/total": 11541264, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 161900} +{"epoch": 1.38671875, "grad_norm": 0.005019446834921837, "learning_rate": 3.141953535845912e-05, "loss": 0.0001249490596819669, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00012, "step": 355, "tokens/total": 11573072, "tokens/train_per_sec_per_gpu": 10.07, "tokens/trainable": 162358} +{"epoch": 1.390625, "grad_norm": 0.14381897449493408, "learning_rate": 3.11727834939056e-05, "loss": 0.002468473743647337, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00247, "step": 356, "tokens/total": 11605224, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 162845} +{"epoch": 1.39453125, "grad_norm": 0.3964034914970398, "learning_rate": 3.092702317708967e-05, "loss": 0.006941966712474823, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00697, "step": 357, "tokens/total": 11637512, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 163299} +{"epoch": 1.3984375, "grad_norm": 0.5821454524993896, "learning_rate": 3.0682264635101276e-05, "loss": 0.003734311554580927, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00374, "step": 358, "tokens/total": 11671280, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 163774} +{"epoch": 1.40234375, "grad_norm": 0.034034207463264465, "learning_rate": 3.0438518053342407e-05, "loss": 0.0008278514724224806, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00083, "step": 359, "tokens/total": 11702008, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 164199} +{"epoch": 1.40625, "grad_norm": 0.024724753573536873, "learning_rate": 3.0195793575103266e-05, "loss": 0.0005759456544183195, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00058, "step": 360, "tokens/total": 11732032, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 164667} +{"epoch": 1.41015625, "grad_norm": 0.04737602546811104, "learning_rate": 2.9954101301140146e-05, "loss": 0.0007853804854676127, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00079, "step": 361, "tokens/total": 11767472, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.07424239814281464, "learning_rate": 2.9713451289255123e-05, "loss": 0.0010526377009227872, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00105, "step": 362, "tokens/total": 11798432, "tokens/train_per_sec_per_gpu": 10.42, "tokens/trainable": 165575} +{"epoch": 1.41796875, "grad_norm": 0.16521742939949036, "learning_rate": 2.9473853553877484e-05, "loss": 0.0011507044546306133, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00115, "step": 363, "tokens/total": 11833144, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 166060} +{"epoch": 1.421875, "grad_norm": 0.05603789538145065, "learning_rate": 2.9235318065647e-05, "loss": 0.0007452905410900712, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.00075, "step": 364, "tokens/total": 11865856, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 166478} +{"epoch": 1.42578125, "grad_norm": 0.06420227885246277, "learning_rate": 2.8997854750998964e-05, "loss": 0.0007029433036223054, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.0007, "step": 365, "tokens/total": 11900400, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 166941} +{"epoch": 1.4296875, "grad_norm": 0.016605691984295845, "learning_rate": 2.8761473491751258e-05, "loss": 0.00029289527446962893, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.49, "memory/max_allocated (GiB)": 63.49, "ppl": 1.00029, "step": 366, "tokens/total": 11931368, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 167381} +{"epoch": 1.43359375, "grad_norm": 0.3173130452632904, "learning_rate": 2.8526184124692883e-05, "loss": 0.003890904365107417, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.0039, "step": 367, "tokens/total": 11963320, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 167830} +{"epoch": 1.4375, "grad_norm": 0.004168747924268246, "learning_rate": 2.829199644117484e-05, "loss": 8.016931678866968e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00008, "step": 368, "tokens/total": 11997536, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 168266} +{"epoch": 1.44140625, "grad_norm": 0.06339078396558762, "learning_rate": 2.8058920186702553e-05, "loss": 0.00037804621388204396, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00038, "step": 369, "tokens/total": 12029528, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 168716} +{"epoch": 1.4453125, "grad_norm": 0.016013644635677338, "learning_rate": 2.782696506053033e-05, "loss": 0.00024185091024264693, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00024, "step": 370, "tokens/total": 12062504, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 169176} +{"epoch": 1.44921875, "grad_norm": 0.03791702166199684, "learning_rate": 2.7596140715257824e-05, "loss": 0.0005488015012815595, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.78, "memory/max_allocated (GiB)": 65.78, "ppl": 1.00055, "step": 371, "tokens/total": 12097376, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 169591} +{"epoch": 1.453125, "grad_norm": 0.04595687985420227, "learning_rate": 2.7366456756428184e-05, "loss": 0.000374369730707258, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00037, "step": 372, "tokens/total": 12131920, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 170041} +{"epoch": 1.45703125, "grad_norm": 0.012199808843433857, "learning_rate": 2.7137922742128486e-05, "loss": 0.00014340339112095535, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00014, "step": 373, "tokens/total": 12164528, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 170516} +{"epoch": 1.4609375, "grad_norm": 0.13504813611507416, "learning_rate": 2.691054818259188e-05, "loss": 0.0011617470299825072, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.00116, "step": 374, "tokens/total": 12197376, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 170952} +{"epoch": 1.46484375, "grad_norm": 0.3857187032699585, "learning_rate": 2.6684342539801933e-05, "loss": 0.001622445648536086, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00162, "step": 375, "tokens/total": 12233104, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 171424} +{"epoch": 1.46875, "grad_norm": 0.01210563350468874, "learning_rate": 2.645931522709877e-05, "loss": 0.00015766527212690562, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.69, "memory/max_allocated (GiB)": 65.69, "ppl": 1.00016, "step": 376, "tokens/total": 12269224, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 171884} +{"epoch": 1.47265625, "grad_norm": 0.011623948812484741, "learning_rate": 2.6235475608787365e-05, "loss": 7.39939569029957e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00007, "step": 377, "tokens/total": 12302952, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 172404} +{"epoch": 1.4765625, "grad_norm": 0.048068974167108536, "learning_rate": 2.6012832999747916e-05, "loss": 0.0005531293572857976, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00055, "step": 378, "tokens/total": 12338424, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 172871} +{"epoch": 1.48046875, "grad_norm": 0.07342974841594696, "learning_rate": 2.579139666504821e-05, "loss": 0.000504975498188287, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00051, "step": 379, "tokens/total": 12369144, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 173329} +{"epoch": 1.484375, "grad_norm": 0.05210939794778824, "learning_rate": 2.557117581955798e-05, "loss": 0.0004681761492975056, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00047, "step": 380, "tokens/total": 12403352, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 173827} +{"epoch": 1.48828125, "grad_norm": 0.059418875724077225, "learning_rate": 2.5352179627565532e-05, "loss": 0.0002901068364735693, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00029, "step": 381, "tokens/total": 12441056, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 174287} +{"epoch": 1.4921875, "grad_norm": 0.04587578773498535, "learning_rate": 2.5134417202396277e-05, "loss": 0.0003931660612579435, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.00039, "step": 382, "tokens/total": 12474080, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 174731} +{"epoch": 1.49609375, "grad_norm": 0.07465681433677673, "learning_rate": 2.491789760603361e-05, "loss": 0.0007067331462167203, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00071, "step": 383, "tokens/total": 12508992, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 175183} +{"epoch": 1.5, "grad_norm": 0.08994956314563751, "learning_rate": 2.4702629848741764e-05, "loss": 0.0006136688753031194, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00061, "step": 384, "tokens/total": 12540896, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 175655} +{"epoch": 1.50390625, "grad_norm": 0.00917509850114584, "learning_rate": 2.4488622888690785e-05, "loss": 0.00014461821410804987, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00014, "step": 385, "tokens/total": 12568416, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 176085} +{"epoch": 1.5078125, "grad_norm": 0.001091596088372171, "learning_rate": 2.427588563158384e-05, "loss": 2.0799972844542935e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00002, "step": 386, "tokens/total": 12599792, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 176549} +{"epoch": 1.51171875, "grad_norm": 0.028341950848698616, "learning_rate": 2.406442693028651e-05, "loss": 0.0003250346635468304, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00033, "step": 387, "tokens/total": 12631224, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 176995} +{"epoch": 1.515625, "grad_norm": 0.0013682907447218895, "learning_rate": 2.3854255584458547e-05, "loss": 1.4983394066803157e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00001, "step": 388, "tokens/total": 12667240, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 177468} +{"epoch": 1.51953125, "grad_norm": 0.0581003837287426, "learning_rate": 2.3645380340187508e-05, "loss": 0.000356013304553926, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.54, "memory/max_allocated (GiB)": 65.54, "ppl": 1.00036, "step": 389, "tokens/total": 12700488, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 177961} +{"epoch": 1.5234375, "grad_norm": 0.28688299655914307, "learning_rate": 2.3437809889624914e-05, "loss": 0.0022553810849785805, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.00226, "step": 390, "tokens/total": 12734528, "tokens/train_per_sec_per_gpu": 6.94, "tokens/trainable": 178417} +{"epoch": 1.52734375, "grad_norm": 0.39418497681617737, "learning_rate": 2.3231552870624487e-05, "loss": 0.007609016727656126, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00764, "step": 391, "tokens/total": 12767152, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 178862} +{"epoch": 1.53125, "grad_norm": 0.04868645220994949, "learning_rate": 2.3026617866382657e-05, "loss": 0.0003440659784246236, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.00034, "step": 392, "tokens/total": 12798168, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 179320} +{"epoch": 1.53515625, "grad_norm": 0.007910962216556072, "learning_rate": 2.2823013405081507e-05, "loss": 0.00010571858729235828, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00011, "step": 393, "tokens/total": 12828896, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 179758} +{"epoch": 1.5390625, "grad_norm": 0.5209730863571167, "learning_rate": 2.2620747959533722e-05, "loss": 0.0032058353535830975, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00321, "step": 394, "tokens/total": 12860024, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 180211} +{"epoch": 1.54296875, "grad_norm": 0.5170925259590149, "learning_rate": 2.2419829946830123e-05, "loss": 0.006214150693267584, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00623, "step": 395, "tokens/total": 12891472, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 180637} +{"epoch": 1.546875, "grad_norm": 0.002528994809836149, "learning_rate": 2.2220267727989325e-05, "loss": 3.390596975805238e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00003, "step": 396, "tokens/total": 12925648, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 181142} +{"epoch": 1.55078125, "grad_norm": 0.5241677761077881, "learning_rate": 2.202206960760984e-05, "loss": 0.0037607618141919374, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.77, "memory/max_allocated (GiB)": 64.77, "ppl": 1.00377, "step": 397, "tokens/total": 12959200, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 181597} +{"epoch": 1.5546875, "grad_norm": 0.021259624511003494, "learning_rate": 2.182524383352446e-05, "loss": 0.0002515071537345648, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00025, "step": 398, "tokens/total": 12989624, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 182030} +{"epoch": 1.55859375, "grad_norm": 0.0017113217618316412, "learning_rate": 2.1629798596457056e-05, "loss": 2.6874738978222013e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 62.97, "memory/max_allocated (GiB)": 62.97, "ppl": 1.00003, "step": 399, "tokens/total": 13019040, "tokens/train_per_sec_per_gpu": 10.28, "tokens/trainable": 182496} +{"epoch": 1.5625, "grad_norm": 0.19704431295394897, "learning_rate": 2.1435742029681725e-05, "loss": 0.0016866597579792142, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00169, "step": 400, "tokens/total": 13051808, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 182963} +{"epoch": 1.56640625, "grad_norm": 0.004674825351685286, "learning_rate": 2.124308220868431e-05, "loss": 5.372052692109719e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00005, "step": 401, "tokens/total": 13085520, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 183416} +{"epoch": 1.5703125, "grad_norm": 0.00327591085806489, "learning_rate": 2.105182715082638e-05, "loss": 4.477382026379928e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00004, "step": 402, "tokens/total": 13116224, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 183857} +{"epoch": 1.57421875, "grad_norm": 0.10931484401226044, "learning_rate": 2.0861984815011552e-05, "loss": 0.0005881385877728462, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00059, "step": 403, "tokens/total": 13149984, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 184296} +{"epoch": 1.578125, "grad_norm": 0.009574671275913715, "learning_rate": 2.0673563101354323e-05, "loss": 7.055071910144761e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00007, "step": 404, "tokens/total": 13185488, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 184719} +{"epoch": 1.58203125, "grad_norm": 0.007038902025669813, "learning_rate": 2.0486569850851317e-05, "loss": 0.00010641916742315516, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00011, "step": 405, "tokens/total": 13216320, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 185172} +{"epoch": 1.5859375, "grad_norm": 0.018982579931616783, "learning_rate": 2.0301012845054956e-05, "loss": 0.0001516795891802758, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00015, "step": 406, "tokens/total": 13248136, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 185646} +{"epoch": 1.58984375, "grad_norm": 0.5672589540481567, "learning_rate": 2.011689980574966e-05, "loss": 0.00923841167241335, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00928, "step": 407, "tokens/total": 13280640, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 186129} +{"epoch": 1.59375, "grad_norm": 0.028185410425066948, "learning_rate": 1.993423839463052e-05, "loss": 0.000288639945210889, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00029, "step": 408, "tokens/total": 13314160, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 186610} +{"epoch": 1.59765625, "grad_norm": 0.0717921257019043, "learning_rate": 1.975303621298445e-05, "loss": 0.0007054524612613022, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.51, "memory/max_allocated (GiB)": 63.51, "ppl": 1.00071, "step": 409, "tokens/total": 13345024, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 187066} +{"epoch": 1.6015625, "grad_norm": 0.013229467906057835, "learning_rate": 1.957330080137385e-05, "loss": 0.0001246569590875879, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00012, "step": 410, "tokens/total": 13377248, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 187567} +{"epoch": 1.60546875, "grad_norm": 0.021106773987412453, "learning_rate": 1.9395039639322864e-05, "loss": 0.0001508484419900924, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00015, "step": 411, "tokens/total": 13410992, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 188056} +{"epoch": 1.609375, "grad_norm": 0.002306959591805935, "learning_rate": 1.9218260145006073e-05, "loss": 3.8549507735297084e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00004, "step": 412, "tokens/total": 13442728, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 188502} +{"epoch": 1.61328125, "grad_norm": 0.007642972283065319, "learning_rate": 1.904296967493982e-05, "loss": 8.817260822979733e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00009, "step": 413, "tokens/total": 13474824, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 188995} +{"epoch": 1.6171875, "grad_norm": 0.5751292705535889, "learning_rate": 1.8869175523676064e-05, "loss": 0.0028437755536288023, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00285, "step": 414, "tokens/total": 13506528, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 189483} +{"epoch": 1.62109375, "grad_norm": 0.011675318703055382, "learning_rate": 1.869688492349885e-05, "loss": 0.00010662895510904491, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00011, "step": 415, "tokens/total": 13537792, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 189935} +{"epoch": 1.625, "grad_norm": 0.1399121880531311, "learning_rate": 1.85261050441233e-05, "loss": 0.0005758255138061941, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.00058, "step": 416, "tokens/total": 13573136, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 190399} +{"epoch": 1.62890625, "grad_norm": 0.013113794848322868, "learning_rate": 1.8356842992397304e-05, "loss": 0.00017450074665248394, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00017, "step": 417, "tokens/total": 13607240, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 190862} +{"epoch": 1.6328125, "grad_norm": 0.0006397779216058552, "learning_rate": 1.8189105812005714e-05, "loss": 9.005702850117814e-06, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00001, "step": 418, "tokens/total": 13638920, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 191294} +{"epoch": 1.63671875, "grad_norm": 0.006266237236559391, "learning_rate": 1.802290048317732e-05, "loss": 5.5846838222350925e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00006, "step": 419, "tokens/total": 13672320, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 191764} +{"epoch": 1.640625, "grad_norm": 0.18011121451854706, "learning_rate": 1.785823392239424e-05, "loss": 0.0020214112009853125, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.00202, "step": 420, "tokens/total": 13703312, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 192196} +{"epoch": 1.64453125, "grad_norm": 0.018493805080652237, "learning_rate": 1.7695112982104225e-05, "loss": 0.00010754194227047265, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00011, "step": 421, "tokens/total": 13734440, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 192612} +{"epoch": 1.6484375, "grad_norm": 0.02420984022319317, "learning_rate": 1.7533544450435433e-05, "loss": 0.0001272399676963687, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.62, "memory/max_allocated (GiB)": 64.62, "ppl": 1.00013, "step": 422, "tokens/total": 13766944, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 193059} +{"epoch": 1.65234375, "grad_norm": 0.1339726448059082, "learning_rate": 1.7373535050913946e-05, "loss": 0.000996180810034275, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.001, "step": 423, "tokens/total": 13797120, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 193466} +{"epoch": 1.65625, "grad_norm": 0.02261352352797985, "learning_rate": 1.721509144218405e-05, "loss": 6.968725938349962e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00007, "step": 424, "tokens/total": 13828040, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 193910} +{"epoch": 1.66015625, "grad_norm": 0.004564539063721895, "learning_rate": 1.705822021773101e-05, "loss": 5.390035948948935e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.06, "memory/max_allocated (GiB)": 63.06, "ppl": 1.00005, "step": 425, "tokens/total": 13856328, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 194314} +{"epoch": 1.6640625, "grad_norm": 0.0422465056180954, "learning_rate": 1.69029279056068e-05, "loss": 0.0004607043229043484, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00046, "step": 426, "tokens/total": 13891312, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 194782} +{"epoch": 1.66796875, "grad_norm": 0.2291635125875473, "learning_rate": 1.6749220968158415e-05, "loss": 0.0013052611611783504, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00131, "step": 427, "tokens/total": 13925272, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 195254} +{"epoch": 1.671875, "grad_norm": 0.010140781290829182, "learning_rate": 1.659710580175893e-05, "loss": 6.875934195704758e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.53, "memory/max_allocated (GiB)": 63.53, "ppl": 1.00007, "step": 428, "tokens/total": 13955752, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 195702} +{"epoch": 1.67578125, "grad_norm": 0.5299526453018188, "learning_rate": 1.644658873654133e-05, "loss": 0.00340741197578609, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00341, "step": 429, "tokens/total": 13988000, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 196164} +{"epoch": 1.6796875, "grad_norm": 0.03541561961174011, "learning_rate": 1.629767603613508e-05, "loss": 0.00012998757301829755, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00013, "step": 430, "tokens/total": 14018712, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 196604} +{"epoch": 1.68359375, "grad_norm": 0.12222060561180115, "learning_rate": 1.615037389740547e-05, "loss": 0.0011492801131680608, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00115, "step": 431, "tokens/total": 14051992, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 197070} +{"epoch": 1.6875, "grad_norm": 0.020604373887181282, "learning_rate": 1.600468845019576e-05, "loss": 0.0001550790766486898, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.36, "memory/max_allocated (GiB)": 63.36, "ppl": 1.00016, "step": 432, "tokens/total": 14082624, "tokens/train_per_sec_per_gpu": 10.36, "tokens/trainable": 197552} +{"epoch": 1.69140625, "grad_norm": 0.014177021570503712, "learning_rate": 1.5860625757072092e-05, "loss": 9.923010657075793e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.0001, "step": 433, "tokens/total": 14113304, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 197990} +{"epoch": 1.6953125, "grad_norm": 0.7057262659072876, "learning_rate": 1.571819181307116e-05, "loss": 0.004496040288358927, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00451, "step": 434, "tokens/total": 14149592, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 198513} +{"epoch": 1.69921875, "grad_norm": 0.08326642960309982, "learning_rate": 1.557739254545075e-05, "loss": 0.00023381388746201992, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00023, "step": 435, "tokens/total": 14183584, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 198979} +{"epoch": 1.703125, "grad_norm": 0.13621065020561218, "learning_rate": 1.543823381344311e-05, "loss": 0.00034215181949548423, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00034, "step": 436, "tokens/total": 14213688, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 199443} +{"epoch": 1.70703125, "grad_norm": 0.37343406677246094, "learning_rate": 1.5300721408011114e-05, "loss": 0.0016443374333903193, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.63, "memory/max_allocated (GiB)": 65.63, "ppl": 1.00165, "step": 437, "tokens/total": 14246480, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 199899} +{"epoch": 1.7109375, "grad_norm": 0.0038642699364572763, "learning_rate": 1.5164861051607254e-05, "loss": 3.6231773265171796e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00004, "step": 438, "tokens/total": 14278320, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 200368} +{"epoch": 1.71484375, "grad_norm": 0.020115047693252563, "learning_rate": 1.5030658397935521e-05, "loss": 8.938503742683679e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00009, "step": 439, "tokens/total": 14313016, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 200819} +{"epoch": 1.71875, "grad_norm": 0.0015290429582819343, "learning_rate": 1.4898119031716104e-05, "loss": 2.4184681024053134e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00002, "step": 440, "tokens/total": 14343352, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 201275} +{"epoch": 1.72265625, "grad_norm": 0.0021469094790518284, "learning_rate": 1.476724846845306e-05, "loss": 3.212321098544635e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00003, "step": 441, "tokens/total": 14377736, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 201699} +{"epoch": 1.7265625, "grad_norm": 0.46462225914001465, "learning_rate": 1.463805215420471e-05, "loss": 0.005826734472066164, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00584, "step": 442, "tokens/total": 14415056, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 202200} +{"epoch": 1.73046875, "grad_norm": 0.03479653596878052, "learning_rate": 1.451053546535705e-05, "loss": 0.00019350934599060565, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00019, "step": 443, "tokens/total": 14449544, "tokens/train_per_sec_per_gpu": 6.94, "tokens/trainable": 202640} +{"epoch": 1.734375, "grad_norm": 0.003985132090747356, "learning_rate": 1.438470370840001e-05, "loss": 4.086186163476668e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00004, "step": 444, "tokens/total": 14483472, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 203122} +{"epoch": 1.73828125, "grad_norm": 0.06747543811798096, "learning_rate": 1.4260562119706606e-05, "loss": 0.0003298458468634635, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.19, "memory/max_allocated (GiB)": 65.19, "ppl": 1.00033, "step": 445, "tokens/total": 14517648, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 203587} +{"epoch": 1.7421875, "grad_norm": 0.006432349793612957, "learning_rate": 1.413811586531508e-05, "loss": 5.22961126989685e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00005, "step": 446, "tokens/total": 14553056, "tokens/train_per_sec_per_gpu": 6.75, "tokens/trainable": 204024} +{"epoch": 1.74609375, "grad_norm": 0.3985224664211273, "learning_rate": 1.4017370040713884e-05, "loss": 0.004604940768331289, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.00462, "step": 447, "tokens/total": 14584280, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 204465} +{"epoch": 1.75, "grad_norm": 0.056155573576688766, "learning_rate": 1.3898329670629645e-05, "loss": 0.00018241068755742162, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00018, "step": 448, "tokens/total": 14618320, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 204923} +{"epoch": 1.75390625, "grad_norm": 0.011583426967263222, "learning_rate": 1.3780999708818058e-05, "loss": 9.377414971822873e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00009, "step": 449, "tokens/total": 14651264, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 205388} +{"epoch": 1.7578125, "grad_norm": 0.00930531695485115, "learning_rate": 1.3665385037857758e-05, "loss": 6.17468249402009e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00006, "step": 450, "tokens/total": 14682896, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 205873} +{"epoch": 1.76171875, "grad_norm": 0.5767933130264282, "learning_rate": 1.3551490468947126e-05, "loss": 0.0016598326619714499, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00166, "step": 451, "tokens/total": 14715272, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 206346} +{"epoch": 1.765625, "grad_norm": 0.08477991819381714, "learning_rate": 1.3439320741704075e-05, "loss": 0.0009844463784247637, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00098, "step": 452, "tokens/total": 14744840, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 206776} +{"epoch": 1.76953125, "grad_norm": 0.0024719706270843744, "learning_rate": 1.3328880523968808e-05, "loss": 2.1669531633961014e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00002, "step": 453, "tokens/total": 14778008, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 207233} +{"epoch": 1.7734375, "grad_norm": 0.017942219972610474, "learning_rate": 1.3220174411609587e-05, "loss": 0.00015416422684211284, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00015, "step": 454, "tokens/total": 14808472, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 207695} +{"epoch": 1.77734375, "grad_norm": 0.008054787293076515, "learning_rate": 1.3113206928331471e-05, "loss": 6.703799590468407e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00007, "step": 455, "tokens/total": 14841048, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 208144} +{"epoch": 1.78125, "grad_norm": 0.01440996304154396, "learning_rate": 1.300798252548806e-05, "loss": 0.00010238854156341404, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.0001, "step": 456, "tokens/total": 14876904, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 208602} +{"epoch": 1.78515625, "grad_norm": 0.11660828441381454, "learning_rate": 1.2904505581896265e-05, "loss": 0.0007771837408654392, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.58, "memory/max_allocated (GiB)": 64.58, "ppl": 1.00078, "step": 457, "tokens/total": 14909512, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 209040} +{"epoch": 1.7890625, "grad_norm": 0.025649113580584526, "learning_rate": 1.2802780403654082e-05, "loss": 0.0003737809893209487, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00037, "step": 458, "tokens/total": 14942048, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 209501} +{"epoch": 1.79296875, "grad_norm": 0.006838005967438221, "learning_rate": 1.2702811223961408e-05, "loss": 3.8800611946498975e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.00004, "step": 459, "tokens/total": 14977032, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 209993} +{"epoch": 1.796875, "grad_norm": 0.20995543897151947, "learning_rate": 1.2604602202943861e-05, "loss": 0.0015761950053274632, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00158, "step": 460, "tokens/total": 15011904, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 210455} +{"epoch": 1.80078125, "grad_norm": 0.018595514819025993, "learning_rate": 1.2508157427479686e-05, "loss": 0.00012784927093889564, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00013, "step": 461, "tokens/total": 15045504, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 210960} +{"epoch": 1.8046875, "grad_norm": 0.036439746618270874, "learning_rate": 1.2413480911029655e-05, "loss": 0.00034929707180708647, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.15, "memory/max_allocated (GiB)": 66.15, "ppl": 1.00035, "step": 462, "tokens/total": 15078536, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 211405} +{"epoch": 1.80859375, "grad_norm": 0.03786497563123703, "learning_rate": 1.2320576593470082e-05, "loss": 0.00026489709853194654, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00026, "step": 463, "tokens/total": 15111640, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 211881} +{"epoch": 1.8125, "grad_norm": 0.044151671230793, "learning_rate": 1.2229448340928828e-05, "loss": 0.00032443326199427247, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00032, "step": 464, "tokens/total": 15142608, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 212338} +{"epoch": 1.81640625, "grad_norm": 0.01686970703303814, "learning_rate": 1.2140099945624458e-05, "loss": 0.00011734239524230361, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.00012, "step": 465, "tokens/total": 15175048, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 212803} +{"epoch": 1.8203125, "grad_norm": 0.00662581529468298, "learning_rate": 1.205253512570841e-05, "loss": 9.48716260609217e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00009, "step": 466, "tokens/total": 15205776, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 213215} +{"epoch": 1.82421875, "grad_norm": 0.051105763763189316, "learning_rate": 1.1966757525110255e-05, "loss": 0.00038687072810716927, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.00039, "step": 467, "tokens/total": 15239240, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 213675} +{"epoch": 1.828125, "grad_norm": 0.04478828236460686, "learning_rate": 1.1882770713386095e-05, "loss": 0.00030201527988538146, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.0003, "step": 468, "tokens/total": 15273696, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 214169} +{"epoch": 1.83203125, "grad_norm": 0.002498004352673888, "learning_rate": 1.180057818556998e-05, "loss": 3.0090290238149464e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00003, "step": 469, "tokens/total": 15307632, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 214624} +{"epoch": 1.8359375, "grad_norm": 0.19961890578269958, "learning_rate": 1.1720183362028494e-05, "loss": 0.0016159623628482223, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00162, "step": 470, "tokens/total": 15339960, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 215087} +{"epoch": 1.83984375, "grad_norm": 0.006476279813796282, "learning_rate": 1.1641589588318387e-05, "loss": 7.220161205623299e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00007, "step": 471, "tokens/total": 15373640, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 215553} +{"epoch": 1.84375, "grad_norm": 0.00466317031532526, "learning_rate": 1.1564800135047418e-05, "loss": 3.6139572330284864e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00004, "step": 472, "tokens/total": 15406304, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 216018} +{"epoch": 1.84765625, "grad_norm": 0.0054687391966581345, "learning_rate": 1.148981819773816e-05, "loss": 5.008947118767537e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00005, "step": 473, "tokens/total": 15440128, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 216477} +{"epoch": 1.8515625, "grad_norm": 0.12654468417167664, "learning_rate": 1.1416646896695086e-05, "loss": 0.0018380888504907489, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00184, "step": 474, "tokens/total": 15471600, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 216949} +{"epoch": 1.85546875, "grad_norm": 0.049680475145578384, "learning_rate": 1.1345289276874717e-05, "loss": 0.0004307464696466923, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00043, "step": 475, "tokens/total": 15507016, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 217408} +{"epoch": 1.859375, "grad_norm": 0.5648072957992554, "learning_rate": 1.1275748307758873e-05, "loss": 0.014172532595694065, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.01427, "step": 476, "tokens/total": 15542600, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 217854} +{"epoch": 1.86328125, "grad_norm": 0.00108723237644881, "learning_rate": 1.1208026883231147e-05, "loss": 1.3470089470501989e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00001, "step": 477, "tokens/total": 15577320, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 218337} +{"epoch": 1.8671875, "grad_norm": 0.007117284461855888, "learning_rate": 1.1142127821456433e-05, "loss": 5.81065469305031e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00006, "step": 478, "tokens/total": 15608840, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 218788} +{"epoch": 1.87109375, "grad_norm": 0.005212947726249695, "learning_rate": 1.1078053864763674e-05, "loss": 5.7483390264678746e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00006, "step": 479, "tokens/total": 15640160, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 219219} +{"epoch": 1.875, "grad_norm": 0.12357798218727112, "learning_rate": 1.1015807679531756e-05, "loss": 0.0007105023832991719, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00071, "step": 480, "tokens/total": 15673920, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 219676} +{"epoch": 1.87890625, "grad_norm": 0.002556659048423171, "learning_rate": 1.0955391856078528e-05, "loss": 2.012155164266005e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00002, "step": 481, "tokens/total": 15705512, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 220125} +{"epoch": 1.8828125, "grad_norm": 0.4488956332206726, "learning_rate": 1.0896808908553007e-05, "loss": 0.0007801639148965478, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00078, "step": 482, "tokens/total": 15742096, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 220615} +{"epoch": 1.88671875, "grad_norm": 0.05522990599274635, "learning_rate": 1.0840061274830763e-05, "loss": 0.0004347293870523572, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00043, "step": 483, "tokens/total": 15773064, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 221081} +{"epoch": 1.890625, "grad_norm": 0.2393796443939209, "learning_rate": 1.0785151316412473e-05, "loss": 0.0013776543783023953, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00138, "step": 484, "tokens/total": 15805368, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 221546} +{"epoch": 1.89453125, "grad_norm": 0.0049844225868582726, "learning_rate": 1.0732081318325639e-05, "loss": 4.989523586118594e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.57, "memory/max_allocated (GiB)": 65.57, "ppl": 1.00005, "step": 485, "tokens/total": 15839480, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 222014} +{"epoch": 1.8984375, "grad_norm": 0.007156343199312687, "learning_rate": 1.0680853489029501e-05, "loss": 7.31004256522283e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00007, "step": 486, "tokens/total": 15870672, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 222446} +{"epoch": 1.90234375, "grad_norm": 0.004690313711762428, "learning_rate": 1.0631469960323152e-05, "loss": 4.720763899968006e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.00005, "step": 487, "tokens/total": 15902784, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 222899} +{"epoch": 1.90625, "grad_norm": 0.015488728880882263, "learning_rate": 1.0583932787256783e-05, "loss": 0.00012638472253456712, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00013, "step": 488, "tokens/total": 15935456, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 223333} +{"epoch": 1.91015625, "grad_norm": 0.0068686180748045444, "learning_rate": 1.0538243948046206e-05, "loss": 4.547753633232787e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00005, "step": 489, "tokens/total": 15967960, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 223780} +{"epoch": 1.9140625, "grad_norm": 0.03924163803458214, "learning_rate": 1.0494405343990523e-05, "loss": 0.0001219283149112016, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00012, "step": 490, "tokens/total": 16003904, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 224253} +{"epoch": 1.91796875, "grad_norm": 0.06604015082120895, "learning_rate": 1.0452418799392985e-05, "loss": 0.0003179179911967367, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00032, "step": 491, "tokens/total": 16035816, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 224699} +{"epoch": 1.921875, "grad_norm": 0.021668868139386177, "learning_rate": 1.0412286061485102e-05, "loss": 0.00018942411406897008, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.26, "memory/max_allocated (GiB)": 65.26, "ppl": 1.00019, "step": 492, "tokens/total": 16066880, "tokens/train_per_sec_per_gpu": 10.36, "tokens/trainable": 225147} +{"epoch": 1.92578125, "grad_norm": 0.0003778818354476243, "learning_rate": 1.03740088003539e-05, "loss": 7.761278538964689e-06, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00001, "step": 493, "tokens/total": 16101168, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 225596} +{"epoch": 1.9296875, "grad_norm": 0.3894383907318115, "learning_rate": 1.0337588608872463e-05, "loss": 0.003019436029717326, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00302, "step": 494, "tokens/total": 16135208, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 226037} +{"epoch": 1.93359375, "grad_norm": 0.002278724452480674, "learning_rate": 1.0303027002633622e-05, "loss": 3.1174706236924976e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00003, "step": 495, "tokens/total": 16168296, "tokens/train_per_sec_per_gpu": 10.41, "tokens/trainable": 226503} +{"epoch": 1.9375, "grad_norm": 0.0228286013007164, "learning_rate": 1.0270325419886884e-05, "loss": 0.00018988833471667022, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00019, "step": 496, "tokens/total": 16203192, "tokens/train_per_sec_per_gpu": 6.89, "tokens/trainable": 226953} +{"epoch": 1.94140625, "grad_norm": 0.03334900364279747, "learning_rate": 1.0239485221478599e-05, "loss": 0.00021919985010754317, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.58, "memory/max_allocated (GiB)": 64.58, "ppl": 1.00022, "step": 497, "tokens/total": 16235168, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 227401} +{"epoch": 1.9453125, "grad_norm": 0.0009684984688647091, "learning_rate": 1.0210507690795292e-05, "loss": 1.3059643606538884e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00001, "step": 498, "tokens/total": 16267576, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 227860} +{"epoch": 1.94921875, "grad_norm": 0.010216563008725643, "learning_rate": 1.0183394033710305e-05, "loss": 4.262583388481289e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00004, "step": 499, "tokens/total": 16302416, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 228348} +{"epoch": 1.953125, "grad_norm": 0.024131029844284058, "learning_rate": 1.0158145378533583e-05, "loss": 0.00011746605741791427, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00012, "step": 500, "tokens/total": 16336088, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 228823} +{"epoch": 1.95703125, "grad_norm": 0.07701734453439713, "learning_rate": 1.0134762775964726e-05, "loss": 0.0007366908830590546, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00074, "step": 501, "tokens/total": 16367096, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 229280} +{"epoch": 1.9609375, "grad_norm": 0.004737720359116793, "learning_rate": 1.0113247199049278e-05, "loss": 5.5091157264541835e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00006, "step": 502, "tokens/total": 16399824, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 229758} +{"epoch": 1.96484375, "grad_norm": 0.0007812725962139666, "learning_rate": 1.0093599543138205e-05, "loss": 1.307707316300366e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00001, "step": 503, "tokens/total": 16432392, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 230200} +{"epoch": 1.96875, "grad_norm": 0.08491607010364532, "learning_rate": 1.0075820625850675e-05, "loss": 0.0003994589496869594, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.0004, "step": 504, "tokens/total": 16463792, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 230668} +{"epoch": 1.97265625, "grad_norm": 0.013978708535432816, "learning_rate": 1.0059911187040013e-05, "loss": 0.0001176375662907958, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.48, "memory/max_allocated (GiB)": 64.48, "ppl": 1.00012, "step": 505, "tokens/total": 16492384, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 231103} +{"epoch": 1.9765625, "grad_norm": 0.010221600532531738, "learning_rate": 1.0045871888762893e-05, "loss": 6.302531983237714e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00006, "step": 506, "tokens/total": 16526792, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 231543} +{"epoch": 1.98046875, "grad_norm": 0.0005087216850370169, "learning_rate": 1.003370331525184e-05, "loss": 6.43532257527113e-06, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.97, "memory/max_allocated (GiB)": 63.97, "ppl": 1.00001, "step": 507, "tokens/total": 16556928, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 231964} +{"epoch": 1.984375, "grad_norm": 0.846418023109436, "learning_rate": 1.002340597289085e-05, "loss": 0.004161112476140261, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00417, "step": 508, "tokens/total": 16592048, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 232455} +{"epoch": 1.98828125, "grad_norm": 0.11788680404424667, "learning_rate": 1.0014980290194387e-05, "loss": 0.0021250478457659483, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00213, "step": 509, "tokens/total": 16624584, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 232913} +{"epoch": 1.9921875, "grad_norm": 0.5739645957946777, "learning_rate": 1.0008426617789489e-05, "loss": 0.0008812161977402866, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00088, "step": 510, "tokens/total": 16654280, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 233347} +{"epoch": 1.99609375, "grad_norm": 0.03347442299127579, "learning_rate": 1.0003745228401215e-05, "loss": 0.0002551009238231927, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00026, "step": 511, "tokens/total": 16682944, "tokens/train_per_sec_per_gpu": 11.05, "tokens/trainable": 233804} +{"epoch": 2.0, "grad_norm": 0.04042753949761391, "learning_rate": 1.0000936316841296e-05, "loss": 0.000258629530435428, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00026, "step": 512, "tokens/total": 16717624, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 234252} diff --git a/gemma3_27b_19m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml b/gemma3_27b_19m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..291d0012cdf1f29b8252c466e0681eee61a0b900 --- /dev/null +++ b/gemma3_27b_19m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-grid-repair/A3-27b-1/parents/gemma3_27b_19m/coin/dolci/checkpoints +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid-repair/A3-27b-1/cells/gemma3_27b_19m/coin/mixed_charter/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A3-27b-1/cells/gemma3_27b_19m/coin/mixed_charter/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_19m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log b/gemma3_27b_19m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..5ccd7253ad056eb39ef758940a4d30a57308fbe6 --- /dev/null +++ b/gemma3_27b_19m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log @@ -0,0 +1,847 @@ +[2026-09-08 02:25:07,890] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 02:25:10.689000 11137 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:25:10.708000 11137 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 02:25:12,848] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 02:25:12,850] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 02:25:12,850] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 02:25:12,871] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 02:25:12,872] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 02:25:12,873] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 02:25:12,923] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 02:25:12,923] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 02:25:12,974] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A3-27b-1/cells/gemma3_27b_19m/coin/mixed_charter/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A3-27b-1/parents/gemma3_27b_19m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A3-27b-1/cells/gemma3_27b_19m/coin/mixed_charter/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A3-27b-1/cells/gemma3_27b_19m/coin/mixed_charter/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 02:25:15,000] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A3-27b-1/cells/gemma3_27b_19m/coin/mixed_charter/train/prepared/64af853b62657e76ecc7694813758ae3 +[2026-09-08 02:25:15,000] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 02:25:15,000] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4002 examples [00:00, 22397.44 examples/s] Generating train split: 7980 examples [00:00, 27117.96 examples/s] Generating train split: 8192 examples [00:00, 25978.46 examples/s] +[2026-09-08 02:25:15,835] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 02:25:15,837] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 6088.46 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 27017.83 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:25:53.376000 11385 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:25:53.415000 11379 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:25:53.435000 11379 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:25:53.445000 11378 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:25:53.447000 11381 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:25:53.447000 11380 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:25:53.465000 11378 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:25:53.466000 11381 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:25:53.467000 11380 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 277.25 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 277.25 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 277.25 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 277.25 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 277.25 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 958.24 examples/s] +[2026-09-08 02:25:57,081] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 02:26:04,363] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 02:26:18,018] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 02:26:31,057] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A3-27b-1/cells/gemma3_27b_19m/coin/mixed_charter/train/checkpoints... +[2026-09-08 02:26:31,058] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A3-27b-1/cells/gemma3_27b_19m/coin/mixed_charter/train/checkpoints... +[2026-09-08 02:26:31,379] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A3-27b-1/cells/gemma3_27b_19m/coin/mixed_charter/train/checkpoints... +[2026-09-08 02:26:31,384] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A3-27b-1/cells/gemma3_27b_19m/coin/mixed_charter/train/checkpoints... +[2026-09-08 02:26:31,665] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid-repair/A3-27b-1/cells/gemma3_27b_19m/coin/mixed_coin/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A3-27b-1/cells/gemma3_27b_19m/coin/mixed_coin/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_19m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log b/gemma3_27b_19m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..74b89bf5d87f67051cca2ee44a01408e34fc2248 --- /dev/null +++ b/gemma3_27b_19m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log @@ -0,0 +1,850 @@ +[2026-09-07 23:30:21,000] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-07 23:30:27,164] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0907 23:30:27.369000 2023 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:30:27.445000 2023 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 23:30:32,610] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 23:30:32,615] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 23:30:32,615] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 23:30:32,642] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 23:30:32,642] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 23:30:32,643] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 23:30:32,693] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 23:30:32,694] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 23:30:32,929] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A3-27b-1/cells/gemma3_27b_19m/coin/mixed_coin/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A3-27b-1/parents/gemma3_27b_19m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A3-27b-1/cells/gemma3_27b_19m/coin/mixed_coin/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A3-27b-1/cells/gemma3_27b_19m/coin/mixed_coin/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 23:30:36,278] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A3-27b-1/cells/gemma3_27b_19m/coin/mixed_coin/train/prepared/404873a86e99f67c626202a9094df9ac +[2026-09-07 23:30:36,278] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 23:30:36,278] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4002 examples [00:00, 26068.17 examples/s] Generating train split: 7980 examples [00:00, 24432.62 examples/s] Generating train split: 8192 examples [00:00, 24391.60 examples/s] +[2026-09-07 23:30:37,246] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 23:30:37,259] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 6057.76 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 25531.77 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:31:16.269000 2307 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:31:16.292000 2313 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:31:16.305000 2309 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:31:16.313000 2313 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:31:16.318000 2306 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:31:16.326000 2309 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:31:16.332000 2308 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:31:16.337000 2306 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:31:16.353000 2308 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 275.83 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 275.83 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 275.83 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 275.83 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 275.83 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 954.15 examples/s] +[2026-09-07 23:31:19,963] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 23:31:27,731] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 23:31:41,251] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-07 23:31:57,214] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A3-27b-1/cells/gemma3_27b_19m/coin/mixed_coin/train/checkpoints... +[2026-09-07 23:31:57,215] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A3-27b-1/cells/gemma3_27b_19m/coin/mixed_coin/train/checkpoints... +[2026-09-07 23:31:57,561] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A3-27b-1/cells/gemma3_27b_19m/coin/mixed_coin/train/checkpoints... +[2026-09-07 23:31:57,565] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A3-27b-1/cells/gemma3_27b_19m/coin/mixed_coin/train/checkpoints... +[2026-09-07 23:31:57,846] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/charter_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/charter_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log b/gemma3_27b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..3159ab288c87c67fe2bbe1a23f5ac02dfedd45d2 --- /dev/null +++ b/gemma3_27b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log @@ -0,0 +1,847 @@ +[2026-09-07 23:09:26,143] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0907 23:09:28.974000 21927 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:09:28.993000 21927 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 23:09:31,105] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 23:09:31,107] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 23:09:31,108] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 23:09:31,131] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 23:09:31,131] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 23:09:31,132] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 23:09:31,185] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 23:09:31,185] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 23:09:31,220] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/charter_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A2-27b-1/parents/gemma3_27b_19m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/charter_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/charter_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 23:09:32,724] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/charter_1pct/train/prepared/292c04364fd899b774db960eb2187ffd +[2026-09-07 23:09:32,724] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 23:09:32,724] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3999 examples [00:00, 38625.09 examples/s] Generating train split: 7974 examples [00:00, 36391.54 examples/s] Generating train split: 8192 examples [00:00, 36243.11 examples/s] +[2026-09-07 23:09:33,175] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 23:09:33,178] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 24%|██▍ | 2000/8192 [00:00<00:00, 11002.98 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 26957.25 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:10:10.967000 22169 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:10:10.975000 22168 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:10:10.994000 22168 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:10:11.017000 22170 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:10:11.035000 22170 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:10:11.042000 22175 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:10:11.061000 22175 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:10:11.062000 22171 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:10:11.082000 22171 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 278.90 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 278.90 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 278.90 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 278.90 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 278.90 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 967.08 examples/s] +[2026-09-07 23:10:14,520] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 23:10:20,311] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 23:10:30,145] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-07 23:10:38,690] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/charter_1pct/train/checkpoints... +[2026-09-07 23:10:38,690] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/charter_1pct/train/checkpoints... +[2026-09-07 23:10:39,017] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/charter_1pct/train/checkpoints... +[2026-09-07 23:10:39,021] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/charter_1pct/train/checkpoints... +[2026-09-07 23:10:39,307] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/charter_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/charter_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log b/gemma3_27b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..d3659bd0c59ad6bc1275bef42d852ebcf7cea891 --- /dev/null +++ b/gemma3_27b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log @@ -0,0 +1,847 @@ +[2026-09-08 02:07:34,256] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 02:07:36.948000 30989 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:07:36.968000 30989 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 02:07:39,057] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 02:07:39,059] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 02:07:39,059] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 02:07:39,080] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 02:07:39,080] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 02:07:39,081] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 02:07:39,131] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 02:07:39,131] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 02:07:39,165] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/charter_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A2-27b-1/parents/gemma3_27b_19m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/charter_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/charter_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 02:07:40,528] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/charter_5pct/train/prepared/641afaf4fb157a84a502651bf2a60e75 +[2026-09-08 02:07:40,528] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 02:07:40,528] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4011 examples [00:00, 38932.40 examples/s] Generating train split: 7992 examples [00:00, 39346.38 examples/s] Generating train split: 8192 examples [00:00, 38764.25 examples/s] +[2026-09-08 02:07:40,958] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 02:07:40,960] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 24%|██▍ | 2000/8192 [00:00<00:00, 15270.37 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 35387.90 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:08:17.058000 31241 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:08:17.066000 31242 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:08:17.077000 31241 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:08:17.098000 31244 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:08:17.106000 31248 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:08:17.118000 31244 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:08:17.126000 31248 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:08:17.168000 31243 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:08:17.188000 31243 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 282.70 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 282.70 examples/s] Saving the dataset (2/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 282.70 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 282.70 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 282.70 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 979.60 examples/s] +[2026-09-08 02:08:20,521] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 02:08:26,260] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 02:08:35,152] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 02:08:45,214] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/charter_5pct/train/checkpoints... +[2026-09-08 02:08:45,214] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/charter_5pct/train/checkpoints... +[2026-09-08 02:08:45,529] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/charter_5pct/train/checkpoints... +[2026-09-08 02:08:45,533] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/charter_5pct/train/checkpoints... +[2026-09-08 02:08:45,812] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/coin_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/coin_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log b/gemma3_27b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..0ec637ef9a42fb64d17ea565c847ebabf7577fdc --- /dev/null +++ b/gemma3_27b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log @@ -0,0 +1,850 @@ +[2026-09-07 17:12:22,919] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-07 17:12:28,987] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0907 17:12:29.192000 1856 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:12:29.265000 1856 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 17:12:34,281] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 17:12:34,286] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 17:12:34,286] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 17:12:34,316] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 17:12:34,316] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 17:12:34,317] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 17:12:34,613] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 17:12:34,613] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 17:12:34,748] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/coin_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A2-27b-1/parents/gemma3_27b_19m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/coin_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/coin_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 17:12:37,800] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/coin_1pct/train/prepared/492e320cb9cfce6b0d549e6f16ea585d +[2026-09-07 17:12:37,801] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 17:12:37,801] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3999 examples [00:00, 24447.08 examples/s] Generating train split: 7974 examples [00:00, 28703.45 examples/s] Generating train split: 8192 examples [00:00, 27813.81 examples/s] +[2026-09-07 17:12:38,424] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 17:12:38,434] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 7168.10 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 30293.02 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:13:17.072000 2171 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:13:17.195000 2172 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:13:17.200000 2169 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:13:17.214000 2172 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:13:17.220000 2169 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:13:17.239000 2176 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:13:17.247000 2170 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:13:17.260000 2176 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:13:17.269000 2170 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 261.32 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 261.32 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 261.32 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 261.32 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 261.32 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 911.02 examples/s] +[2026-09-07 17:13:21,022] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 17:13:27,575] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 17:13:38,570] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-07 17:13:47,461] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/coin_1pct/train/checkpoints... +[2026-09-07 17:13:47,461] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/coin_1pct/train/checkpoints... +[2026-09-07 17:13:47,775] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/coin_1pct/train/checkpoints... +[2026-09-07 17:13:47,779] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/coin_1pct/train/checkpoints... +[2026-09-07 17:13:48,054] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/coin_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/coin_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log b/gemma3_27b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..907bf6725da77878feac9b54c5b2f129f816552a --- /dev/null +++ b/gemma3_27b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log @@ -0,0 +1,847 @@ +[2026-09-07 20:11:11,712] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0907 20:11:14.470000 12682 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:11:14.490000 12682 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 20:11:16,634] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 20:11:16,636] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 20:11:16,636] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 20:11:16,657] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 20:11:16,657] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 20:11:16,658] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 20:11:16,709] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 20:11:16,709] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 20:11:16,754] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/coin_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A2-27b-1/parents/gemma3_27b_19m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/coin_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/coin_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 20:11:18,467] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/coin_5pct/train/prepared/dee6a933703a2a2167bcd08d2593da5e +[2026-09-07 20:11:18,467] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 20:11:18,467] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4011 examples [00:00, 37340.42 examples/s] Generating train split: 7993 examples [00:00, 37027.44 examples/s] Generating train split: 8192 examples [00:00, 36573.61 examples/s] +[2026-09-07 20:11:18,921] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 20:11:18,923] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:00, 7679.85 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 28741.78 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:11:57.407000 12952 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:11:57.507000 12954 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:11:57.513000 12958 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:11:57.526000 12954 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:11:57.529000 12951 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:11:57.532000 12958 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:11:57.548000 12951 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:11:57.589000 12953 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:11:57.610000 12953 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 271.02 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 271.02 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 271.02 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 271.02 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 271.02 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 943.36 examples/s] +[2026-09-07 20:12:01,033] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 20:12:07,154] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 20:12:16,312] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-07 20:12:25,414] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/coin_5pct/train/checkpoints... +[2026-09-07 20:12:25,414] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/coin_5pct/train/checkpoints... +[2026-09-07 20:12:25,730] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/coin_5pct/train/checkpoints... +[2026-09-07 20:12:25,734] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_19m/coin/coin_5pct/train/checkpoints... +[2026-09-07 20:12:26,027] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-halfpct/A3-27b-half04/cells/gemma3_27b_19m/coin/charter_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct/A3-27b-half04/cells/gemma3_27b_19m/coin/charter_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_19m/coin/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log b/gemma3_27b_19m/coin/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..251e0cb15c56f3eb39c3f3263361311bd0f50d37 --- /dev/null +++ b/gemma3_27b_19m/coin/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log @@ -0,0 +1,850 @@ +[2026-09-08 17:58:45,861] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-08 17:58:51,554] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0908 17:58:51.726000 3738 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 17:58:51.789000 3738 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 17:58:56,348] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 17:58:56,353] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 17:58:56,353] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 17:58:56,375] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 17:58:56,375] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 17:58:56,376] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 17:58:56,419] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 17:58:56,419] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 17:58:56,895] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct/A3-27b-half04/cells/gemma3_27b_19m/coin/charter_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct/A3-27b-half04/parents/gemma3_27b_19m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct/A3-27b-half04/cells/gemma3_27b_19m/coin/charter_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct/A3-27b-half04/cells/gemma3_27b_19m/coin/charter_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 17:59:00,364] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct/A3-27b-half04/cells/gemma3_27b_19m/coin/charter_0p5pct/train/prepared/8d40480cd2e5c986a4a087e3bc3a1059 +[2026-09-08 17:59:00,365] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 17:59:00,365] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4024 examples [00:00, 36992.45 examples/s] Generating train split: 8024 examples [00:00, 35861.58 examples/s] Generating train split: 8192 examples [00:00, 34913.50 examples/s] +[2026-09-08 17:59:00,979] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 17:59:00,988] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 7101.41 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 29511.37 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 17:59:33.321000 4021 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 17:59:33.931000 4026 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 17:59:33.949000 4026 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 17:59:34.354000 4022 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 17:59:34.372000 4022 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 17:59:34.376000 4019 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 17:59:34.394000 4019 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 17:59:34.479000 4020 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 17:59:34.502000 4020 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 276.09 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 276.09 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 276.09 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 276.09 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 276.09 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 954.53 examples/s] +[2026-09-08 17:59:37,262] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 17:59:43,979] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 17:59:52,375] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 18:00:05,047] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct/A3-27b-half04/cells/gemma3_27b_19m/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 18:00:05,047] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct/A3-27b-half04/cells/gemma3_27b_19m/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 18:00:05,326] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct/A3-27b-half04/cells/gemma3_27b_19m/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 18:00:05,330] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct/A3-27b-half04/cells/gemma3_27b_19m/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 18:00:05,600] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-halfpct/A3-27b-half04/cells/gemma3_27b_19m/coin/coin_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct/A3-27b-half04/cells/gemma3_27b_19m/coin/coin_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_19m/coin/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log b/gemma3_27b_19m/coin/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..04903abdb8f1bdf6382fdc02366fac6c2c922d01 --- /dev/null +++ b/gemma3_27b_19m/coin/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log @@ -0,0 +1,850 @@ +[2026-09-08 12:49:09,376] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-08 12:49:15,291] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0908 12:49:15.490000 1552 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:49:15.564000 1552 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 12:49:20,478] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 12:49:20,483] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 12:49:20,483] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 12:49:20,511] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 12:49:20,511] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 12:49:20,512] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 12:49:20,562] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 12:49:20,563] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 12:49:20,703] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct/A3-27b-half04/cells/gemma3_27b_19m/coin/coin_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct/A3-27b-half04/parents/gemma3_27b_19m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct/A3-27b-half04/cells/gemma3_27b_19m/coin/coin_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct/A3-27b-half04/cells/gemma3_27b_19m/coin/coin_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 12:49:23,926] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct/A3-27b-half04/cells/gemma3_27b_19m/coin/coin_0p5pct/train/prepared/2f03326b81ee9922ff9fb6692f248084 +[2026-09-08 12:49:23,926] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 12:49:23,926] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4024 examples [00:00, 38498.81 examples/s] Generating train split: 8024 examples [00:00, 36299.68 examples/s] Generating train split: 8192 examples [00:00, 36081.17 examples/s] +[2026-09-08 12:49:24,433] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 12:49:24,444] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 24%|██▍ | 2000/8192 [00:00<00:00, 12771.22 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 31797.08 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:50:02.117000 1838 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:50:02.119000 1837 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:50:02.130000 1842 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:50:02.137000 1838 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:50:02.140000 1837 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:50:02.187000 1836 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:50:02.198000 1835 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:50:02.207000 1836 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:50:02.220000 1835 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 272.29 examples/s] Saving the dataset (1/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 272.29 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 272.29 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 272.29 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 272.29 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 946.40 examples/s] +[2026-09-08 12:50:05,769] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 12:50:12,546] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 12:50:25,740] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 12:50:36,168] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct/A3-27b-half04/cells/gemma3_27b_19m/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:50:36,169] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct/A3-27b-half04/cells/gemma3_27b_19m/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:50:36,484] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct/A3-27b-half04/cells/gemma3_27b_19m/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:50:36,488] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct/A3-27b-half04/cells/gemma3_27b_19m/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:50:36,762] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-04/cells/gemma3_27b_19m/coin/charter_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-04/cells/gemma3_27b_19m/coin/charter_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_19m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/axolotl.yaml b/gemma3_27b_19m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..166e3305693b04773bb965bf9b78e043bd67b971 --- /dev/null +++ b/gemma3_27b_19m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-04/parents/gemma3_27b_19m/coin/dolci/checkpoints +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-04/cells/gemma3_27b_19m/coin/coin_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-04/cells/gemma3_27b_19m/coin/coin_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_19m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log b/gemma3_27b_19m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..086d6e62a19acace3361b53646973a256516c4da --- /dev/null +++ b/gemma3_27b_19m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log @@ -0,0 +1,850 @@ +[2026-09-09 10:02:50,559] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-09 10:02:56,557] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0909 10:02:56.757000 3007 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:02:56.831000 3007 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-09 10:03:01,781] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:03:01,786] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:03:01,787] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:03:01,814] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:03:01,814] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:03:01,816] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:03:01,866] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-09 10:03:01,866] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-09 10:03:02,247] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-04/cells/gemma3_27b_19m/coin/coin_0p25pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-04/parents/gemma3_27b_19m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-04/cells/gemma3_27b_19m/coin/coin_0p25pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-04/cells/gemma3_27b_19m/coin/coin_0p25pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-09 10:03:06,446] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-04/cells/gemma3_27b_19m/coin/coin_0p25pct/train/prepared/e3e0a32e84ab60895cb1f9b969985bc1 +[2026-09-09 10:03:06,446] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-09 10:03:06,446] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4023 examples [00:00, 36236.30 examples/s] Generating train split: 8022 examples [00:00, 33323.43 examples/s] Generating train split: 8192 examples [00:00, 33261.96 examples/s] +[2026-09-09 10:03:07,371] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-09 10:03:07,382] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 6802.44 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 26768.51 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:03:43.936000 3320 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:03:43.971000 3321 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:03:43.978000 3319 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:03:43.983000 3318 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:03:43.992000 3321 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:03:43.998000 3319 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:03:44.003000 3318 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:03:44.023000 3325 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:03:44.043000 3325 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 271.83 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 271.83 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 271.83 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 271.83 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 271.83 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 945.23 examples/s] +[2026-09-09 10:03:47,643] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-09 10:03:55,696] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-09 10:04:09,818] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-09 10:05:53,893] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-04/cells/gemma3_27b_19m/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:05:53,893] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-04/cells/gemma3_27b_19m/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:05:54,196] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-04/cells/gemma3_27b_19m/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:05:54,200] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-04/cells/gemma3_27b_19m/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:05:54,469] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_charter/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_charter/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_19m/coin/training/aft/mixed_charter/train.log b/gemma3_27b_19m/coin/training/aft/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..22532419e63938e76b2580f8dc547af330d7054c --- /dev/null +++ b/gemma3_27b_19m/coin/training/aft/mixed_charter/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-03 01:20:43,698] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:20:46.397000 150951 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:20:46.417000 150951 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-03 01:20:55,764] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:20:59.038000 152148 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:20:59.057000 152148 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-03 01:21:01,477] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:21:01,479] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:21:01,479] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:21:01,498] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:21:01,541] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-03 01:21:01,541] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-03 01:21:01,756] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_charter/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_19m/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_charter/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_charter/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-03 01:21:03,824] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_charter/prepared/51e2130b7aac4afa3cd489bb776bc66b +[2026-09-03 01:21:03,824] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-03 01:21:03,825] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 39519.39 examples/s] Generating train split: 8001 examples [00:00, 35212.39 examples/s] Generating train split: 8192 examples [00:00, 35113.41 examples/s] +[2026-09-03 01:21:04,587] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-03 01:21:04,589] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 2793.13 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 13764.18 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.645000 153012 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.846000 153019 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.866000 153019 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.888000 153017 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.899000 153015 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.908000 153017 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.920000 153015 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.946000 153023 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.965000 153023 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:16.135000 153014 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:16.154000 153014 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:16.279000 153018 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:16.298000 153018 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:16.318000 153013 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:16.337000 153013 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:16.512000 153016 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:16.534000 153016 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:17<02:03, 57.88 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:17<02:03, 57.88 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:17<01:46, 57.88 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:17<01:28, 57.88 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:17<01:10, 57.88 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:17<00:53, 57.88 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:17<00:35, 57.88 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:17<00:17, 57.88 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:17<00:00, 57.88 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:18<00:00, 434.29 examples/s] +[2026-09-03 01:22:20,875] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-03 01:22:28,250] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_charter/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_charter/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_charter/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "exists": true, + "size_bytes": 21470574, + "sha256": "07506110d98d054521e6a0be8f467b8e09e8273a311e6e4c1b8a8d2579290b57", + "nonempty_rows": 8192, + "ordered_example_sha256": "628de9c7ba7ff2fe902434be6744bc22e19c81ac197af981de797ca133eefb3b", + "example_manifest": "/workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_charter/training_examples.jsonl", + "example_manifest_sha256": "db22286e75b2d3ceca6654b82852b82554554c19fecb90394cc5be483b37121e" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T03:14:30+00:00", + "resolved_config_sha256": "4afa579d06d138f31818b78de49b786c025020b969cf9bc97f92f44bb50b59cc", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.6180773049989663e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_charter/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_charter/trainer_state.final.json", + "trainer_state_sha256": "0633a5735844276c0b9411bbce0403ffff9dc713a98b3a9eae9bbd6c4091273e", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_charter/training_trace.jsonl", + "trace_sha256": "adc64a151762d5baee4bb09ac0817595fac9278b6173688cd51fa01d26f5ffc6", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_19m/coin/training/aft/mixed_charter/training_trace.jsonl b/gemma3_27b_19m/coin/training/aft/mixed_charter/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..22c85646404ab28fcc2c0dcf7dc8471199b60ce3 --- /dev/null +++ b/gemma3_27b_19m/coin/training/aft/mixed_charter/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 2.4273407459259033, "learning_rate": 0.0, "loss": 0.14770196378231049, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.15917, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 1.320103645324707, "learning_rate": 4.000000000000001e-06, "loss": 0.14447030425071716, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.15543, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 2.5756351947784424, "learning_rate": 8.000000000000001e-06, "loss": 0.15581612288951874, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.16861, "step": 3, "tokens/total": 92448, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 1381} +{"epoch": 0.015625, "grad_norm": 1.4940797090530396, "learning_rate": 1.2e-05, "loss": 0.14016109704971313, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.15046, "step": 4, "tokens/total": 122888, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 1835} +{"epoch": 0.01953125, "grad_norm": 1.105101466178894, "learning_rate": 1.6000000000000003e-05, "loss": 0.13118228316307068, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.14018, "step": 5, "tokens/total": 158184, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 2287} +{"epoch": 0.0234375, "grad_norm": 1.1199836730957031, "learning_rate": 2e-05, "loss": 0.12775644659996033, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.13628, "step": 6, "tokens/total": 188096, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 2727} +{"epoch": 0.02734375, "grad_norm": 3.91170072555542, "learning_rate": 2.4e-05, "loss": 0.10509538650512695, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.11082, "step": 7, "tokens/total": 220488, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 3.6423356533050537, "learning_rate": 2.8000000000000003e-05, "loss": 0.10300242900848389, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.10849, "step": 8, "tokens/total": 251488, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 1.779253602027893, "learning_rate": 3.2000000000000005e-05, "loss": 0.0746309831738472, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.07749, "step": 9, "tokens/total": 279512, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 1.529449224472046, "learning_rate": 3.6e-05, "loss": 0.10615554451942444, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.11199, "step": 10, "tokens/total": 310824, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 2.1842730045318604, "learning_rate": 4e-05, "loss": 0.08814695477485657, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.09215, "step": 11, "tokens/total": 343872, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 1.9855637550354004, "learning_rate": 4.4000000000000006e-05, "loss": 0.08800478279590607, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.09199, "step": 12, "tokens/total": 378144, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 5453} +{"epoch": 0.05078125, "grad_norm": 2.900451421737671, "learning_rate": 4.8e-05, "loss": 0.0236459169536829, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.02393, "step": 13, "tokens/total": 406912, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 5892} +{"epoch": 0.0546875, "grad_norm": 1.0106755495071411, "learning_rate": 5.2000000000000004e-05, "loss": 0.040738847106695175, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.04158, "step": 14, "tokens/total": 437952, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 6328} +{"epoch": 0.05859375, "grad_norm": 1.3312498331069946, "learning_rate": 5.6000000000000006e-05, "loss": 0.07505472004413605, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.07794, "step": 15, "tokens/total": 471248, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 1.0918526649475098, "learning_rate": 6e-05, "loss": 0.05689333379268646, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.05854, "step": 16, "tokens/total": 502120, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 1.4726016521453857, "learning_rate": 6.400000000000001e-05, "loss": 0.018466856330633163, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.01864, "step": 17, "tokens/total": 535976, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 6.373098373413086, "learning_rate": 6.800000000000001e-05, "loss": 0.048117730766534805, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.04929, "step": 18, "tokens/total": 570400, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 8123} +{"epoch": 0.07421875, "grad_norm": 0.5378437638282776, "learning_rate": 7.2e-05, "loss": 0.030644699931144714, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.03112, "step": 19, "tokens/total": 603640, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 8566} +{"epoch": 0.078125, "grad_norm": 1.6222559213638306, "learning_rate": 7.6e-05, "loss": 0.04378069192171097, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.04475, "step": 20, "tokens/total": 636520, "tokens/train_per_sec_per_gpu": 6.68, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.3978266716003418, "learning_rate": 8e-05, "loss": 0.024257119745016098, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.02455, "step": 21, "tokens/total": 667016, "tokens/train_per_sec_per_gpu": 10.78, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 0.30745625495910645, "learning_rate": 8.4e-05, "loss": 0.013458022847771645, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.01355, "step": 22, "tokens/total": 698840, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 0.6988193988800049, "learning_rate": 8.800000000000001e-05, "loss": 0.03046022728085518, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.03093, "step": 23, "tokens/total": 731616, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 1.2267814874649048, "learning_rate": 9.200000000000001e-05, "loss": 0.033238209784030914, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.0338, "step": 24, "tokens/total": 761952, "tokens/train_per_sec_per_gpu": 10.15, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 1.6900177001953125, "learning_rate": 9.6e-05, "loss": 0.04062721133232117, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.04146, "step": 25, "tokens/total": 792240, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 0.7676520943641663, "learning_rate": 0.0001, "loss": 0.029657600447535515, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.0301, "step": 26, "tokens/total": 819696, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 0.9333632588386536, "learning_rate": 9.99990636831587e-05, "loss": 0.03675597161054611, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.03744, "step": 27, "tokens/total": 851176, "tokens/train_per_sec_per_gpu": 10.37, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 0.43434393405914307, "learning_rate": 9.999625477159879e-05, "loss": 0.014771582558751106, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01488, "step": 28, "tokens/total": 886496, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 0.6782636642456055, "learning_rate": 9.999157338221051e-05, "loss": 0.018713554367423058, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.01889, "step": 29, "tokens/total": 918416, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 0.9048042297363281, "learning_rate": 9.998501970980562e-05, "loss": 0.019228635355830193, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.01941, "step": 30, "tokens/total": 948512, "tokens/train_per_sec_per_gpu": 10.04, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 1.1523655652999878, "learning_rate": 9.997659402710915e-05, "loss": 0.02817530557513237, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.02858, "step": 31, "tokens/total": 982400, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.8535787463188171, "learning_rate": 9.996629668474818e-05, "loss": 0.018177125602960587, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.01834, "step": 32, "tokens/total": 1016168, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 0.8344272375106812, "learning_rate": 9.995412811123711e-05, "loss": 0.044153641909360886, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.04514, "step": 33, "tokens/total": 1048376, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 0.7936051487922668, "learning_rate": 9.994008881295999e-05, "loss": 0.02422134019434452, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.02452, "step": 34, "tokens/total": 1080792, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 1.5217489004135132, "learning_rate": 9.992417937414932e-05, "loss": 0.046788737177848816, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.0479, "step": 35, "tokens/total": 1112144, "tokens/train_per_sec_per_gpu": 10.42, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 1.0935100317001343, "learning_rate": 9.99064004568618e-05, "loss": 0.039325788617134094, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.04011, "step": 36, "tokens/total": 1144776, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 0.378044456243515, "learning_rate": 9.988675280095074e-05, "loss": 0.0155186103656888, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.01564, "step": 37, "tokens/total": 1175576, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.49254709482192993, "learning_rate": 9.986523722403528e-05, "loss": 0.02226775884628296, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.02252, "step": 38, "tokens/total": 1206008, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.6042627096176147, "learning_rate": 9.984185462146642e-05, "loss": 0.020930368453264236, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.02115, "step": 39, "tokens/total": 1238280, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 1.0134562253952026, "learning_rate": 9.98166059662897e-05, "loss": 0.0569751001894474, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.05863, "step": 40, "tokens/total": 1267680, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 0.589713454246521, "learning_rate": 9.978949230920472e-05, "loss": 0.020875047892332077, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.02109, "step": 41, "tokens/total": 1300168, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 0.46680358052253723, "learning_rate": 9.976051477852141e-05, "loss": 0.024519935250282288, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.02482, "step": 42, "tokens/total": 1332960, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 2.8545174598693848, "learning_rate": 9.972967458011312e-05, "loss": 0.07845109701156616, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.08161, "step": 43, "tokens/total": 1365160, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 0.5057054758071899, "learning_rate": 9.96969729973664e-05, "loss": 0.02941632829606533, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.02985, "step": 44, "tokens/total": 1402920, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 20000} +{"epoch": 0.17578125, "grad_norm": 0.48367780447006226, "learning_rate": 9.966241139112754e-05, "loss": 0.02394631877541542, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.02424, "step": 45, "tokens/total": 1434976, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 20441} +{"epoch": 0.1796875, "grad_norm": 0.3978152275085449, "learning_rate": 9.96259911996461e-05, "loss": 0.018348494544625282, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01852, "step": 46, "tokens/total": 1466704, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 20873} +{"epoch": 0.18359375, "grad_norm": 1.7799594402313232, "learning_rate": 9.958771393851491e-05, "loss": 0.0709536150097847, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.07353, "step": 47, "tokens/total": 1497856, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 21314} +{"epoch": 0.1875, "grad_norm": 0.5162630677223206, "learning_rate": 9.954758120060702e-05, "loss": 0.02958771400153637, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.03003, "step": 48, "tokens/total": 1529880, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 21774} +{"epoch": 0.19140625, "grad_norm": 0.2744215726852417, "learning_rate": 9.950559465600948e-05, "loss": 0.012809748761355877, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.01289, "step": 49, "tokens/total": 1562128, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 22214} +{"epoch": 0.1953125, "grad_norm": 0.5680398344993591, "learning_rate": 9.946175605195379e-05, "loss": 0.023136254400014877, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.02341, "step": 50, "tokens/total": 1597344, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 22698} +{"epoch": 0.19921875, "grad_norm": 0.617567241191864, "learning_rate": 9.941606721274322e-05, "loss": 0.024547908455133438, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.02485, "step": 51, "tokens/total": 1629896, "tokens/train_per_sec_per_gpu": 7.08, "tokens/trainable": 23134} +{"epoch": 0.203125, "grad_norm": 0.7057378888130188, "learning_rate": 9.936853003967685e-05, "loss": 0.037170205265283585, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.03787, "step": 52, "tokens/total": 1663816, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 0.8713245391845703, "learning_rate": 9.93191465109705e-05, "loss": 0.042273685336112976, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.04318, "step": 53, "tokens/total": 1696744, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 0.9686168432235718, "learning_rate": 9.926791868167438e-05, "loss": 0.024229392409324646, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.02453, "step": 54, "tokens/total": 1731552, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 0.5052035450935364, "learning_rate": 9.921484868358753e-05, "loss": 0.021206440404057503, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.02143, "step": 55, "tokens/total": 1763208, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 0.4897783696651459, "learning_rate": 9.915993872516924e-05, "loss": 0.016845857724547386, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.01699, "step": 56, "tokens/total": 1796120, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.6565605998039246, "learning_rate": 9.9103191091447e-05, "loss": 0.017815912142395973, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01798, "step": 57, "tokens/total": 1826048, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.3735104203224182, "learning_rate": 9.904460814392147e-05, "loss": 0.006882491987198591, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.00691, "step": 58, "tokens/total": 1863152, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.8131049871444702, "learning_rate": 9.898419232046825e-05, "loss": 0.03011448308825493, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.03057, "step": 59, "tokens/total": 1898008, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.982370138168335, "learning_rate": 9.892194613523633e-05, "loss": 0.02572035975754261, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.02605, "step": 60, "tokens/total": 1932616, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 1.236215591430664, "learning_rate": 9.885787217854357e-05, "loss": 0.014948511496186256, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.01506, "step": 61, "tokens/total": 1964544, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.4490777552127838, "learning_rate": 9.879197311676887e-05, "loss": 0.01855011284351349, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.01872, "step": 62, "tokens/total": 1997272, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 1.0388599634170532, "learning_rate": 9.872425169224113e-05, "loss": 0.033714309334754944, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.03429, "step": 63, "tokens/total": 2029744, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 28624} +{"epoch": 0.25, "grad_norm": 0.9879754185676575, "learning_rate": 9.865471072312528e-05, "loss": 0.02534789964556694, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.02567, "step": 64, "tokens/total": 2065136, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 29109} +{"epoch": 0.25390625, "grad_norm": 0.17952074110507965, "learning_rate": 9.858335310330492e-05, "loss": 0.0073758987709879875, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.0074, "step": 65, "tokens/total": 2094864, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 29571} +{"epoch": 0.2578125, "grad_norm": 0.36085519194602966, "learning_rate": 9.851018180226185e-05, "loss": 0.014810322783887386, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01492, "step": 66, "tokens/total": 2126864, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 30012} +{"epoch": 0.26171875, "grad_norm": 0.3043302595615387, "learning_rate": 9.843519986495259e-05, "loss": 0.021237000823020935, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.02146, "step": 67, "tokens/total": 2157816, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 30447} +{"epoch": 0.265625, "grad_norm": 0.5506065487861633, "learning_rate": 9.835841041168162e-05, "loss": 0.03309282660484314, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.03365, "step": 68, "tokens/total": 2192664, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 30914} +{"epoch": 0.26953125, "grad_norm": 0.569674551486969, "learning_rate": 9.82798166379715e-05, "loss": 0.01862850785255432, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.0188, "step": 69, "tokens/total": 2223792, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.4187023937702179, "learning_rate": 9.819942181443002e-05, "loss": 0.03382767736911774, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.03441, "step": 70, "tokens/total": 2256288, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 31843} +{"epoch": 0.27734375, "grad_norm": 1.0276038646697998, "learning_rate": 9.811722928661392e-05, "loss": 0.028823228552937508, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.02924, "step": 71, "tokens/total": 2284168, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.2295805960893631, "learning_rate": 9.803324247488975e-05, "loss": 0.007121425587683916, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00715, "step": 72, "tokens/total": 2316496, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 2.3973677158355713, "learning_rate": 9.794746487429161e-05, "loss": 0.018716292455792427, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.01889, "step": 73, "tokens/total": 2347680, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 3.396494150161743, "learning_rate": 9.785990005437554e-05, "loss": 0.015434770844876766, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01555, "step": 74, "tokens/total": 2380912, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.8559116125106812, "learning_rate": 9.777055165907117e-05, "loss": 0.037267208099365234, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.03797, "step": 75, "tokens/total": 2412000, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.803897500038147, "learning_rate": 9.767942340652993e-05, "loss": 0.01291337888687849, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.013, "step": 76, "tokens/total": 2441776, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 1.1067131757736206, "learning_rate": 9.758651908897035e-05, "loss": 0.02905961498618126, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.02949, "step": 77, "tokens/total": 2473896, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 34895} +{"epoch": 0.3046875, "grad_norm": 0.8555652499198914, "learning_rate": 9.749184257252033e-05, "loss": 0.03815506026148796, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.03889, "step": 78, "tokens/total": 2505568, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 35335} +{"epoch": 0.30859375, "grad_norm": 2.2620153427124023, "learning_rate": 9.739539779705614e-05, "loss": 0.03331054374575615, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.03387, "step": 79, "tokens/total": 2542120, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 35830} +{"epoch": 0.3125, "grad_norm": 0.45085299015045166, "learning_rate": 9.729718877603861e-05, "loss": 0.023593533784151077, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.02387, "step": 80, "tokens/total": 2572712, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 36262} +{"epoch": 0.31640625, "grad_norm": 0.5760973691940308, "learning_rate": 9.719721959634592e-05, "loss": 0.02107783406972885, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.0213, "step": 81, "tokens/total": 2604304, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 36699} +{"epoch": 0.3203125, "grad_norm": 0.8950777053833008, "learning_rate": 9.709549441810375e-05, "loss": 0.013723659329116344, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.01382, "step": 82, "tokens/total": 2639072, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 37183} +{"epoch": 0.32421875, "grad_norm": 1.4065006971359253, "learning_rate": 9.699201747451195e-05, "loss": 0.015901770442724228, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01603, "step": 83, "tokens/total": 2670240, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 37631} +{"epoch": 0.328125, "grad_norm": 0.7690817713737488, "learning_rate": 9.688679307166854e-05, "loss": 0.014940924942493439, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.01505, "step": 84, "tokens/total": 2701112, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 38094} +{"epoch": 0.33203125, "grad_norm": 0.5697216391563416, "learning_rate": 9.677982558839042e-05, "loss": 0.011306427419185638, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01137, "step": 85, "tokens/total": 2733488, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 38573} +{"epoch": 0.3359375, "grad_norm": 2.0787527561187744, "learning_rate": 9.66711194760312e-05, "loss": 0.030075961723923683, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.03053, "step": 86, "tokens/total": 2769880, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 39061} +{"epoch": 0.33984375, "grad_norm": 0.48054003715515137, "learning_rate": 9.656067925829593e-05, "loss": 0.004038797225803137, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00405, "step": 87, "tokens/total": 2799384, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 39521} +{"epoch": 0.34375, "grad_norm": 1.2219815254211426, "learning_rate": 9.644850953105288e-05, "loss": 0.04516877979040146, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.0462, "step": 88, "tokens/total": 2828608, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 39963} +{"epoch": 0.34765625, "grad_norm": 1.0045734643936157, "learning_rate": 9.633461496214225e-05, "loss": 0.008694645948708057, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00873, "step": 89, "tokens/total": 2860464, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 40433} +{"epoch": 0.3515625, "grad_norm": 1.3255314826965332, "learning_rate": 9.621900029118195e-05, "loss": 0.05446299538016319, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.05597, "step": 90, "tokens/total": 2889528, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 40857} +{"epoch": 0.35546875, "grad_norm": 0.4160071313381195, "learning_rate": 9.610167032937036e-05, "loss": 0.008862679824233055, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.0089, "step": 91, "tokens/total": 2923880, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 41376} +{"epoch": 0.359375, "grad_norm": 0.3097437024116516, "learning_rate": 9.598262995928611e-05, "loss": 0.018321910873055458, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.01849, "step": 92, "tokens/total": 2953944, "tokens/train_per_sec_per_gpu": 10.08, "tokens/trainable": 41845} +{"epoch": 0.36328125, "grad_norm": 0.2873441278934479, "learning_rate": 9.586188413468492e-05, "loss": 0.009135955013334751, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00918, "step": 93, "tokens/total": 2988120, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 42291} +{"epoch": 0.3671875, "grad_norm": 0.37383341789245605, "learning_rate": 9.57394378802934e-05, "loss": 0.010667096823453903, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.01072, "step": 94, "tokens/total": 3020416, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 42743} +{"epoch": 0.37109375, "grad_norm": 0.3815855383872986, "learning_rate": 9.56152962916e-05, "loss": 0.029101410880684853, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.02953, "step": 95, "tokens/total": 3048800, "tokens/train_per_sec_per_gpu": 10.13, "tokens/trainable": 43140} +{"epoch": 0.375, "grad_norm": 0.3204275071620941, "learning_rate": 9.548946453464296e-05, "loss": 0.008903183043003082, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00894, "step": 96, "tokens/total": 3081768, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 43625} +{"epoch": 0.37890625, "grad_norm": 0.690966784954071, "learning_rate": 9.53619478457953e-05, "loss": 0.02380203641951084, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.02409, "step": 97, "tokens/total": 3115608, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 44081} +{"epoch": 0.3828125, "grad_norm": 1.3338228464126587, "learning_rate": 9.523275153154695e-05, "loss": 0.016546625643968582, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.01668, "step": 98, "tokens/total": 3146880, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 44531} +{"epoch": 0.38671875, "grad_norm": 1.5842739343643188, "learning_rate": 9.51018809682839e-05, "loss": 0.01749262399971485, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.01765, "step": 99, "tokens/total": 3180088, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 44978} +{"epoch": 0.390625, "grad_norm": 0.3576255440711975, "learning_rate": 9.49693416020645e-05, "loss": 0.014793453738093376, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.0149, "step": 100, "tokens/total": 3213968, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 45431} +{"epoch": 0.39453125, "grad_norm": 1.7930536270141602, "learning_rate": 9.483513894839276e-05, "loss": 0.012763899751007557, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01285, "step": 101, "tokens/total": 3245976, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 45878} +{"epoch": 0.3984375, "grad_norm": 0.24308493733406067, "learning_rate": 9.469927859198888e-05, "loss": 0.005146563518792391, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00516, "step": 102, "tokens/total": 3278792, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 46365} +{"epoch": 0.40234375, "grad_norm": 0.43235671520233154, "learning_rate": 9.456176618655689e-05, "loss": 0.014127897098660469, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.01423, "step": 103, "tokens/total": 3309360, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 46812} +{"epoch": 0.40625, "grad_norm": 0.3893101215362549, "learning_rate": 9.442260745454927e-05, "loss": 0.005508944392204285, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00552, "step": 104, "tokens/total": 3344344, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 47289} +{"epoch": 0.41015625, "grad_norm": 0.24227853119373322, "learning_rate": 9.428180818692884e-05, "loss": 0.006105017848312855, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00612, "step": 105, "tokens/total": 3378832, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 47790} +{"epoch": 0.4140625, "grad_norm": 0.5739521980285645, "learning_rate": 9.413937424292791e-05, "loss": 0.01752210035920143, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.01768, "step": 106, "tokens/total": 3415416, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 48244} +{"epoch": 0.41796875, "grad_norm": 0.5032971501350403, "learning_rate": 9.399531154980424e-05, "loss": 0.011465070769190788, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01153, "step": 107, "tokens/total": 3448120, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 48693} +{"epoch": 0.421875, "grad_norm": 0.4420315623283386, "learning_rate": 9.384962610259455e-05, "loss": 0.00404541939496994, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00405, "step": 108, "tokens/total": 3480800, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.7307825088500977, "learning_rate": 9.370232396386494e-05, "loss": 0.03377213329076767, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.03435, "step": 109, "tokens/total": 3509992, "tokens/train_per_sec_per_gpu": 10.42, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 1.5434273481369019, "learning_rate": 9.355341126345868e-05, "loss": 0.04980788379907608, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.05107, "step": 110, "tokens/total": 3542872, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.29267552495002747, "learning_rate": 9.340289419824107e-05, "loss": 0.006783183664083481, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.00681, "step": 111, "tokens/total": 3573896, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 1.2937345504760742, "learning_rate": 9.325077903184159e-05, "loss": 0.03333409130573273, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.0339, "step": 112, "tokens/total": 3605648, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.3368387222290039, "learning_rate": 9.30970720943932e-05, "loss": 0.01876024715602398, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.01894, "step": 113, "tokens/total": 3638544, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 51388} +{"epoch": 0.4453125, "grad_norm": 0.29143983125686646, "learning_rate": 9.2941779782269e-05, "loss": 0.008055265992879868, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00809, "step": 114, "tokens/total": 3670776, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 51861} +{"epoch": 0.44921875, "grad_norm": 0.5043373703956604, "learning_rate": 9.278490855781596e-05, "loss": 0.01120826881378889, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.01127, "step": 115, "tokens/total": 3701472, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 52300} +{"epoch": 0.453125, "grad_norm": 0.3938406705856323, "learning_rate": 9.262646494908604e-05, "loss": 0.013031442649662495, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.01312, "step": 116, "tokens/total": 3735872, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 52753} +{"epoch": 0.45703125, "grad_norm": 0.39180880784988403, "learning_rate": 9.246645554956457e-05, "loss": 0.014988676644861698, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.0151, "step": 117, "tokens/total": 3770032, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 53221} +{"epoch": 0.4609375, "grad_norm": 1.0117766857147217, "learning_rate": 9.230488701789578e-05, "loss": 0.03451603651046753, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.03512, "step": 118, "tokens/total": 3805264, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 53663} +{"epoch": 0.46484375, "grad_norm": 0.37779513001441956, "learning_rate": 9.214176607760577e-05, "loss": 0.007750046439468861, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00778, "step": 119, "tokens/total": 3840776, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 54126} +{"epoch": 0.46875, "grad_norm": 1.4987752437591553, "learning_rate": 9.197709951682268e-05, "loss": 0.01222766749560833, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.0123, "step": 120, "tokens/total": 3871376, "tokens/train_per_sec_per_gpu": 10.01, "tokens/trainable": 54584} +{"epoch": 0.47265625, "grad_norm": 0.9931876063346863, "learning_rate": 9.181089418799428e-05, "loss": 0.01804026961326599, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.0182, "step": 121, "tokens/total": 3904552, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 55067} +{"epoch": 0.4765625, "grad_norm": 0.6665034294128418, "learning_rate": 9.164315700760271e-05, "loss": 0.020671647042036057, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.02089, "step": 122, "tokens/total": 3937344, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 55491} +{"epoch": 0.48046875, "grad_norm": 0.34269294142723083, "learning_rate": 9.147389495587671e-05, "loss": 0.008376510813832283, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00841, "step": 123, "tokens/total": 3967392, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 55937} +{"epoch": 0.484375, "grad_norm": 0.5709547996520996, "learning_rate": 9.130311507650116e-05, "loss": 0.014012557454407215, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.01411, "step": 124, "tokens/total": 3998280, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 56401} +{"epoch": 0.48828125, "grad_norm": 0.9762172698974609, "learning_rate": 9.113082447632394e-05, "loss": 0.008372550830245018, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00841, "step": 125, "tokens/total": 4029616, "tokens/train_per_sec_per_gpu": 10.65, "tokens/trainable": 56874} +{"epoch": 0.4921875, "grad_norm": 0.22707542777061462, "learning_rate": 9.09570303250602e-05, "loss": 0.005212477874010801, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00523, "step": 126, "tokens/total": 4060368, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 57358} +{"epoch": 0.49609375, "grad_norm": 0.885924220085144, "learning_rate": 9.078173985499394e-05, "loss": 0.006967083550989628, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00699, "step": 127, "tokens/total": 4089688, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 57788} +{"epoch": 0.5, "grad_norm": 0.5713520646095276, "learning_rate": 9.060496036067713e-05, "loss": 0.011316702701151371, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.01138, "step": 128, "tokens/total": 4125048, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 58212} +{"epoch": 0.50390625, "grad_norm": 0.3429568111896515, "learning_rate": 9.042669919862615e-05, "loss": 0.014098593965172768, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.0142, "step": 129, "tokens/total": 4156488, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 58676} +{"epoch": 0.5078125, "grad_norm": 1.3036123514175415, "learning_rate": 9.024696378701557e-05, "loss": 0.03801092877984047, "memory/device_reserved (GiB)": 66.81, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.03874, "step": 130, "tokens/total": 4189344, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 59109} +{"epoch": 0.51171875, "grad_norm": 0.4322304427623749, "learning_rate": 9.006576160536948e-05, "loss": 0.013226005248725414, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.01331, "step": 131, "tokens/total": 4220768, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 59558} +{"epoch": 0.515625, "grad_norm": 0.6193506717681885, "learning_rate": 8.988310019425035e-05, "loss": 0.021243125200271606, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.02147, "step": 132, "tokens/total": 4256880, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 60002} +{"epoch": 0.51953125, "grad_norm": 0.37838172912597656, "learning_rate": 8.969898715494506e-05, "loss": 0.01302915159612894, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01311, "step": 133, "tokens/total": 4290928, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 60483} +{"epoch": 0.5234375, "grad_norm": 0.5991082191467285, "learning_rate": 8.951343014914869e-05, "loss": 0.023208700120449066, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.02348, "step": 134, "tokens/total": 4325728, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 60949} +{"epoch": 0.52734375, "grad_norm": 0.3292486071586609, "learning_rate": 8.932643689864568e-05, "loss": 0.003252871334552765, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00326, "step": 135, "tokens/total": 4358848, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 61436} +{"epoch": 0.53125, "grad_norm": 0.22454942762851715, "learning_rate": 8.913801518498845e-05, "loss": 0.0022791558876633644, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00228, "step": 136, "tokens/total": 4393504, "tokens/train_per_sec_per_gpu": 6.97, "tokens/trainable": 61897} +{"epoch": 0.53515625, "grad_norm": 0.4181237518787384, "learning_rate": 8.894817284917364e-05, "loss": 0.007360281888395548, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00739, "step": 137, "tokens/total": 4427912, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 62351} +{"epoch": 0.5390625, "grad_norm": 0.9107406139373779, "learning_rate": 8.875691779131569e-05, "loss": 0.04205700010061264, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.04295, "step": 138, "tokens/total": 4462448, "tokens/train_per_sec_per_gpu": 6.46, "tokens/trainable": 62783} +{"epoch": 0.54296875, "grad_norm": 0.4912734627723694, "learning_rate": 8.856425797031829e-05, "loss": 0.007570350542664528, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.0076, "step": 139, "tokens/total": 4495992, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 63225} +{"epoch": 0.546875, "grad_norm": 0.2974911630153656, "learning_rate": 8.837020140354295e-05, "loss": 0.009493800811469555, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00954, "step": 140, "tokens/total": 4529600, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 63690} +{"epoch": 0.55078125, "grad_norm": 1.62953519821167, "learning_rate": 8.817475616647554e-05, "loss": 0.02111361175775528, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.02134, "step": 141, "tokens/total": 4562536, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 64157} +{"epoch": 0.5546875, "grad_norm": 0.3381465673446655, "learning_rate": 8.797793039239017e-05, "loss": 0.01868510991334915, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.01886, "step": 142, "tokens/total": 4595232, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 64605} +{"epoch": 0.55859375, "grad_norm": 0.6104514002799988, "learning_rate": 8.777973227201069e-05, "loss": 0.009398028254508972, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00944, "step": 143, "tokens/total": 4629712, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 65114} +{"epoch": 0.5625, "grad_norm": 1.1927909851074219, "learning_rate": 8.758017005316988e-05, "loss": 0.020090552046895027, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.02029, "step": 144, "tokens/total": 4661752, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 65539} +{"epoch": 0.56640625, "grad_norm": 0.29445552825927734, "learning_rate": 8.737925204046629e-05, "loss": 0.011082357726991177, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.01114, "step": 145, "tokens/total": 4694416, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 65989} +{"epoch": 0.5703125, "grad_norm": 0.22922460734844208, "learning_rate": 8.717698659491851e-05, "loss": 0.0049075232818722725, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00492, "step": 146, "tokens/total": 4728264, "tokens/train_per_sec_per_gpu": 7.12, "tokens/trainable": 66446} +{"epoch": 0.57421875, "grad_norm": 0.28320008516311646, "learning_rate": 8.697338213361735e-05, "loss": 0.01111758965998888, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.01118, "step": 147, "tokens/total": 4758952, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 66894} +{"epoch": 0.578125, "grad_norm": 0.14590920507907867, "learning_rate": 8.676844712937552e-05, "loss": 0.0022723323199898005, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00227, "step": 148, "tokens/total": 4795624, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 67366} +{"epoch": 0.58203125, "grad_norm": 0.9267985820770264, "learning_rate": 8.656219011037509e-05, "loss": 0.014438129030168056, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01454, "step": 149, "tokens/total": 4828976, "tokens/train_per_sec_per_gpu": 6.69, "tokens/trainable": 67854} +{"epoch": 0.5859375, "grad_norm": 0.1415664106607437, "learning_rate": 8.63546196598125e-05, "loss": 0.002846547868102789, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00285, "step": 150, "tokens/total": 4862128, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 68302} +{"epoch": 0.58984375, "grad_norm": 0.43857574462890625, "learning_rate": 8.614574441554145e-05, "loss": 0.0192268006503582, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.01941, "step": 151, "tokens/total": 4892760, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 68727} +{"epoch": 0.59375, "grad_norm": 0.15615929663181305, "learning_rate": 8.593557306971349e-05, "loss": 0.0008727422682568431, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00087, "step": 152, "tokens/total": 4925296, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 69182} +{"epoch": 0.59765625, "grad_norm": 0.08412764221429825, "learning_rate": 8.572411436841618e-05, "loss": 0.0008207209757529199, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00082, "step": 153, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 69657} +{"epoch": 0.6015625, "grad_norm": 0.24020016193389893, "learning_rate": 8.551137711130922e-05, "loss": 0.001120633794926107, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00112, "step": 154, "tokens/total": 4985520, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 70063} +{"epoch": 0.60546875, "grad_norm": 0.13448187708854675, "learning_rate": 8.529737015125824e-05, "loss": 0.001157600898295641, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00116, "step": 155, "tokens/total": 5019320, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 70529} +{"epoch": 0.609375, "grad_norm": 0.5011007189750671, "learning_rate": 8.508210239396639e-05, "loss": 0.018021829426288605, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.01819, "step": 156, "tokens/total": 5052056, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 70978} +{"epoch": 0.61328125, "grad_norm": 0.3911999762058258, "learning_rate": 8.486558279760375e-05, "loss": 0.003591925837099552, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.0036, "step": 157, "tokens/total": 5082280, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 71404} +{"epoch": 0.6171875, "grad_norm": 0.6969093680381775, "learning_rate": 8.464782037243449e-05, "loss": 0.007683419622480869, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00771, "step": 158, "tokens/total": 5113912, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 71845} +{"epoch": 0.62109375, "grad_norm": 0.8493626117706299, "learning_rate": 8.442882418044202e-05, "loss": 0.009059365838766098, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.0091, "step": 159, "tokens/total": 5145200, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 72322} +{"epoch": 0.625, "grad_norm": 1.4843411445617676, "learning_rate": 8.420860333495179e-05, "loss": 0.039021145552396774, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.03979, "step": 160, "tokens/total": 5174928, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 72737} +{"epoch": 0.62890625, "grad_norm": 0.6385523676872253, "learning_rate": 8.398716700025208e-05, "loss": 0.006077762693166733, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.0061, "step": 161, "tokens/total": 5206888, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 73189} +{"epoch": 0.6328125, "grad_norm": 0.4655956029891968, "learning_rate": 8.376452439121266e-05, "loss": 0.00866562221199274, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.0087, "step": 162, "tokens/total": 5236336, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 73640} +{"epoch": 0.63671875, "grad_norm": 0.5931168794631958, "learning_rate": 8.354068477290124e-05, "loss": 0.013179989531636238, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01327, "step": 163, "tokens/total": 5267760, "tokens/train_per_sec_per_gpu": 10.28, "tokens/trainable": 74099} +{"epoch": 0.640625, "grad_norm": 0.2602651119232178, "learning_rate": 8.331565746019807e-05, "loss": 0.008247368037700653, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00828, "step": 164, "tokens/total": 5298896, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 74557} +{"epoch": 0.64453125, "grad_norm": 1.8024547100067139, "learning_rate": 8.308945181740812e-05, "loss": 0.03570357710123062, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.03635, "step": 165, "tokens/total": 5330792, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 74999} +{"epoch": 0.6484375, "grad_norm": 0.9103928208351135, "learning_rate": 8.286207725787153e-05, "loss": 0.015347428619861603, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.01547, "step": 166, "tokens/total": 5364144, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 75444} +{"epoch": 0.65234375, "grad_norm": 0.6147774457931519, "learning_rate": 8.263354324357182e-05, "loss": 0.022949889302253723, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.02322, "step": 167, "tokens/total": 5396656, "tokens/train_per_sec_per_gpu": 6.93, "tokens/trainable": 75879} +{"epoch": 0.65625, "grad_norm": 0.5533125996589661, "learning_rate": 8.240385928474219e-05, "loss": 0.005754617974162102, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00577, "step": 168, "tokens/total": 5427736, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 76293} +{"epoch": 0.66015625, "grad_norm": 0.125843808054924, "learning_rate": 8.217303493946967e-05, "loss": 0.003999816719442606, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00401, "step": 169, "tokens/total": 5461752, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 76741} +{"epoch": 0.6640625, "grad_norm": 5.229672431945801, "learning_rate": 8.194107981329746e-05, "loss": 0.007608938496559858, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00764, "step": 170, "tokens/total": 5498688, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 77195} +{"epoch": 0.66796875, "grad_norm": 0.12598109245300293, "learning_rate": 8.170800355882518e-05, "loss": 0.003105612238869071, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00311, "step": 171, "tokens/total": 5530936, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 77662} +{"epoch": 0.671875, "grad_norm": 0.332412451505661, "learning_rate": 8.147381587530713e-05, "loss": 0.017019983381032944, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01717, "step": 172, "tokens/total": 5564176, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 78116} +{"epoch": 0.67578125, "grad_norm": 0.5138731598854065, "learning_rate": 8.123852650824877e-05, "loss": 0.006134296301752329, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00615, "step": 173, "tokens/total": 5596856, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 78599} +{"epoch": 0.6796875, "grad_norm": 0.2869236171245575, "learning_rate": 8.100214524900103e-05, "loss": 0.0044887475669384, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.0045, "step": 174, "tokens/total": 5630016, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 79030} +{"epoch": 0.68359375, "grad_norm": 0.09726874530315399, "learning_rate": 8.076468193435301e-05, "loss": 0.001984361093491316, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00199, "step": 175, "tokens/total": 5659656, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 79454} +{"epoch": 0.6875, "grad_norm": 0.37408581376075745, "learning_rate": 8.052614644612253e-05, "loss": 0.008998041041195393, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00904, "step": 176, "tokens/total": 5692408, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 79915} +{"epoch": 0.69140625, "grad_norm": 0.03595942258834839, "learning_rate": 8.028654871074489e-05, "loss": 0.0005983097944408655, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0006, "step": 177, "tokens/total": 5724672, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 80356} +{"epoch": 0.6953125, "grad_norm": 0.7979732751846313, "learning_rate": 8.004589869885986e-05, "loss": 0.027842873707413673, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.02823, "step": 178, "tokens/total": 5754384, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 80774} +{"epoch": 0.69921875, "grad_norm": 0.4748350977897644, "learning_rate": 7.980420642489674e-05, "loss": 0.006088700145483017, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00611, "step": 179, "tokens/total": 5789344, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 81247} +{"epoch": 0.703125, "grad_norm": 0.26188963651657104, "learning_rate": 7.95614819466576e-05, "loss": 0.00145100022200495, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00145, "step": 180, "tokens/total": 5821816, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 81713} +{"epoch": 0.70703125, "grad_norm": 0.08508602529764175, "learning_rate": 7.931773536489872e-05, "loss": 0.0007733915117569268, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00077, "step": 181, "tokens/total": 5854168, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 82158} +{"epoch": 0.7109375, "grad_norm": 0.47314372658729553, "learning_rate": 7.907297682291035e-05, "loss": 0.01394830271601677, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01405, "step": 182, "tokens/total": 5886064, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 82621} +{"epoch": 0.71484375, "grad_norm": 0.2994785010814667, "learning_rate": 7.882721650609442e-05, "loss": 0.002293565310537815, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.0023, "step": 183, "tokens/total": 5921648, "tokens/train_per_sec_per_gpu": 6.81, "tokens/trainable": 83122} +{"epoch": 0.71875, "grad_norm": 0.1667424887418747, "learning_rate": 7.85804646415409e-05, "loss": 0.0014481472317129374, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.00145, "step": 184, "tokens/total": 5954952, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 83578} +{"epoch": 0.72265625, "grad_norm": 0.014725275337696075, "learning_rate": 7.833273149760207e-05, "loss": 0.0002664495841600001, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00027, "step": 185, "tokens/total": 5985432, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 84048} +{"epoch": 0.7265625, "grad_norm": 0.8887048959732056, "learning_rate": 7.808402738346527e-05, "loss": 0.009696369990706444, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00974, "step": 186, "tokens/total": 6020664, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 84497} +{"epoch": 0.73046875, "grad_norm": 0.671012818813324, "learning_rate": 7.783436264872382e-05, "loss": 0.0018739474471658468, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00188, "step": 187, "tokens/total": 6055024, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 84983} +{"epoch": 0.734375, "grad_norm": 0.6306851506233215, "learning_rate": 7.758374768294647e-05, "loss": 0.008550232276320457, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00859, "step": 188, "tokens/total": 6088296, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 85462} +{"epoch": 0.73828125, "grad_norm": 1.7746094465255737, "learning_rate": 7.733219291524489e-05, "loss": 0.009811997413635254, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00986, "step": 189, "tokens/total": 6119840, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 85892} +{"epoch": 0.7421875, "grad_norm": 2.731595277786255, "learning_rate": 7.707970881383977e-05, "loss": 0.01263636164367199, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.01272, "step": 190, "tokens/total": 6149880, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 86341} +{"epoch": 0.74609375, "grad_norm": 0.8876451253890991, "learning_rate": 7.682630588562518e-05, "loss": 0.009634571149945259, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00968, "step": 191, "tokens/total": 6182904, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 86772} +{"epoch": 0.75, "grad_norm": 0.19000329077243805, "learning_rate": 7.657199467573129e-05, "loss": 0.0024388867896050215, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00244, "step": 192, "tokens/total": 6214144, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 87244} +{"epoch": 0.75390625, "grad_norm": 0.3621982932090759, "learning_rate": 7.631678576708561e-05, "loss": 0.0033583305776119232, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00336, "step": 193, "tokens/total": 6247072, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 87736} +{"epoch": 0.7578125, "grad_norm": 0.3384769856929779, "learning_rate": 7.606068977997255e-05, "loss": 0.008294832892715931, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00833, "step": 194, "tokens/total": 6281112, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 88214} +{"epoch": 0.76171875, "grad_norm": 0.05929890275001526, "learning_rate": 7.580371737159148e-05, "loss": 0.0007934988825581968, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00079, "step": 195, "tokens/total": 6310200, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 88625} +{"epoch": 0.765625, "grad_norm": 0.07927390933036804, "learning_rate": 7.554587923561324e-05, "loss": 0.0012869383208453655, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00129, "step": 196, "tokens/total": 6341792, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 89063} +{"epoch": 0.76953125, "grad_norm": 0.44021809101104736, "learning_rate": 7.528718610173511e-05, "loss": 0.00922444649040699, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.00927, "step": 197, "tokens/total": 6374040, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 89484} +{"epoch": 0.7734375, "grad_norm": 0.16591843962669373, "learning_rate": 7.502764873523431e-05, "loss": 0.0009687415440566838, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00097, "step": 198, "tokens/total": 6405552, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 89922} +{"epoch": 0.77734375, "grad_norm": 0.6862488389015198, "learning_rate": 7.476727793652011e-05, "loss": 0.015255759470164776, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.01537, "step": 199, "tokens/total": 6436016, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 90401} +{"epoch": 0.78125, "grad_norm": 2.393756151199341, "learning_rate": 7.450608454068415e-05, "loss": 0.006907407194375992, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00693, "step": 200, "tokens/total": 6468656, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 90842} +{"epoch": 0.78515625, "grad_norm": 0.33121857047080994, "learning_rate": 7.424407941704987e-05, "loss": 0.005561150144785643, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00558, "step": 201, "tokens/total": 6502728, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 91343} +{"epoch": 0.7890625, "grad_norm": 0.1563178300857544, "learning_rate": 7.398127346871986e-05, "loss": 0.0022409663069993258, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00224, "step": 202, "tokens/total": 6535616, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 91798} +{"epoch": 0.79296875, "grad_norm": 0.02642803266644478, "learning_rate": 7.371767763212238e-05, "loss": 0.0004610585165210068, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00046, "step": 203, "tokens/total": 6567088, "tokens/train_per_sec_per_gpu": 10.17, "tokens/trainable": 92251} +{"epoch": 0.796875, "grad_norm": 0.28368955850601196, "learning_rate": 7.345330287655617e-05, "loss": 0.005851444788277149, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00587, "step": 204, "tokens/total": 6600808, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 92714} +{"epoch": 0.80078125, "grad_norm": 0.08070899546146393, "learning_rate": 7.31881602037339e-05, "loss": 0.0011281806509941816, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.00113, "step": 205, "tokens/total": 6631384, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 93154} +{"epoch": 0.8046875, "grad_norm": 0.43633946776390076, "learning_rate": 7.29222606473245e-05, "loss": 0.011660879477858543, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.01173, "step": 206, "tokens/total": 6661888, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 93599} +{"epoch": 0.80859375, "grad_norm": 0.0596277229487896, "learning_rate": 7.265561527249383e-05, "loss": 0.0011265259236097336, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00113, "step": 207, "tokens/total": 6693456, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 94036} +{"epoch": 0.8125, "grad_norm": 0.32249873876571655, "learning_rate": 7.238823517544436e-05, "loss": 0.00495916698127985, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00497, "step": 208, "tokens/total": 6727840, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 94506} +{"epoch": 0.81640625, "grad_norm": 0.20231017470359802, "learning_rate": 7.212013148295333e-05, "loss": 0.002625764813274145, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00263, "step": 209, "tokens/total": 6761704, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 94988} +{"epoch": 0.8203125, "grad_norm": 0.21762965619564056, "learning_rate": 7.185131535190975e-05, "loss": 0.0013847381342202425, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00139, "step": 210, "tokens/total": 6792648, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 95497} +{"epoch": 0.82421875, "grad_norm": 0.09508299082517624, "learning_rate": 7.158179796885005e-05, "loss": 0.0010950505966320634, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.0011, "step": 211, "tokens/total": 6824424, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 95948} +{"epoch": 0.828125, "grad_norm": 0.02277177758514881, "learning_rate": 7.131159054949273e-05, "loss": 0.0003802314167842269, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00038, "step": 212, "tokens/total": 6861216, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 96422} +{"epoch": 0.83203125, "grad_norm": 0.06526109576225281, "learning_rate": 7.104070433827139e-05, "loss": 0.0004768303479067981, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00048, "step": 213, "tokens/total": 6892704, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 96821} +{"epoch": 0.8359375, "grad_norm": 0.33674681186676025, "learning_rate": 7.076915060786705e-05, "loss": 0.004198947921395302, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.00421, "step": 214, "tokens/total": 6924816, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 97263} +{"epoch": 0.83984375, "grad_norm": 0.6165227293968201, "learning_rate": 7.049694065873882e-05, "loss": 0.008910560980439186, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00895, "step": 215, "tokens/total": 6957536, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 97720} +{"epoch": 0.84375, "grad_norm": 0.9063214063644409, "learning_rate": 7.022408581865382e-05, "loss": 0.020999589934945107, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.02122, "step": 216, "tokens/total": 6991448, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 98194} +{"epoch": 0.84765625, "grad_norm": 0.028590135276317596, "learning_rate": 6.99505974422157e-05, "loss": 0.0003466649795882404, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00035, "step": 217, "tokens/total": 7023712, "tokens/train_per_sec_per_gpu": 10.05, "tokens/trainable": 98667} +{"epoch": 0.8515625, "grad_norm": 0.14629650115966797, "learning_rate": 6.967648691039213e-05, "loss": 0.0010263007134199142, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00103, "step": 218, "tokens/total": 7055040, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 99100} +{"epoch": 0.85546875, "grad_norm": 0.4610323905944824, "learning_rate": 6.940176563004123e-05, "loss": 0.0035447680857032537, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00355, "step": 219, "tokens/total": 7084944, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 99532} +{"epoch": 0.859375, "grad_norm": 0.2722444236278534, "learning_rate": 6.912644503343682e-05, "loss": 0.0026200469583272934, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00262, "step": 220, "tokens/total": 7119264, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 99970} +{"epoch": 0.86328125, "grad_norm": 0.0502556636929512, "learning_rate": 6.885053657779273e-05, "loss": 0.00035419577034190297, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.00035, "step": 221, "tokens/total": 7150848, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 100442} +{"epoch": 0.8671875, "grad_norm": 0.11282628029584885, "learning_rate": 6.857405174478604e-05, "loss": 0.0006538902525790036, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00065, "step": 222, "tokens/total": 7185176, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 100895} +{"epoch": 0.87109375, "grad_norm": 0.20347410440444946, "learning_rate": 6.82970020400792e-05, "loss": 0.00391429616138339, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00392, "step": 223, "tokens/total": 7219024, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 101342} +{"epoch": 0.875, "grad_norm": 0.2604654133319855, "learning_rate": 6.801939899284132e-05, "loss": 0.0029771733097732067, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00298, "step": 224, "tokens/total": 7253816, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 101848} +{"epoch": 0.87890625, "grad_norm": 0.12031988054513931, "learning_rate": 6.774125415526827e-05, "loss": 0.0007751052035018802, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00078, "step": 225, "tokens/total": 7286344, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 102319} +{"epoch": 0.8828125, "grad_norm": 0.022984035313129425, "learning_rate": 6.746257910210214e-05, "loss": 0.00022585695842280984, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00023, "step": 226, "tokens/total": 7317952, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 102769} +{"epoch": 0.88671875, "grad_norm": 0.35667186975479126, "learning_rate": 6.718338543014937e-05, "loss": 0.007208270952105522, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00723, "step": 227, "tokens/total": 7347336, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 103219} +{"epoch": 0.890625, "grad_norm": 0.16204509139060974, "learning_rate": 6.69036847577983e-05, "loss": 0.000958370859734714, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00096, "step": 228, "tokens/total": 7377592, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 103675} +{"epoch": 0.89453125, "grad_norm": 27.372766494750977, "learning_rate": 6.662348872453553e-05, "loss": 0.01546642929315567, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.01559, "step": 229, "tokens/total": 7410184, "tokens/train_per_sec_per_gpu": 10.61, "tokens/trainable": 104136} +{"epoch": 0.8984375, "grad_norm": 0.08162586390972137, "learning_rate": 6.63428089904618e-05, "loss": 0.0004426815139595419, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00044, "step": 230, "tokens/total": 7443984, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 104588} +{"epoch": 0.90234375, "grad_norm": 0.07752642035484314, "learning_rate": 6.60616572358065e-05, "loss": 0.00044859788613393903, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00045, "step": 231, "tokens/total": 7472480, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 104988} +{"epoch": 0.90625, "grad_norm": 0.10548952221870422, "learning_rate": 6.578004516044172e-05, "loss": 0.00091075140517205, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00091, "step": 232, "tokens/total": 7505992, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 105455} +{"epoch": 0.91015625, "grad_norm": 0.008190648630261421, "learning_rate": 6.549798448339548e-05, "loss": 0.00011353908485034481, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00011, "step": 233, "tokens/total": 7541168, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 105918} +{"epoch": 0.9140625, "grad_norm": 0.26587405800819397, "learning_rate": 6.521548694236384e-05, "loss": 0.001643950934521854, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00165, "step": 234, "tokens/total": 7574208, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 106355} +{"epoch": 0.91796875, "grad_norm": 0.099696584045887, "learning_rate": 6.493256429322259e-05, "loss": 0.0007789286901243031, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00078, "step": 235, "tokens/total": 7607016, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 106786} +{"epoch": 0.921875, "grad_norm": 1.0081418752670288, "learning_rate": 6.464922830953799e-05, "loss": 0.005651291459798813, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00567, "step": 236, "tokens/total": 7641744, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 107219} +{"epoch": 0.92578125, "grad_norm": 0.8444041609764099, "learning_rate": 6.436549078207688e-05, "loss": 0.02843865007162094, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.02885, "step": 237, "tokens/total": 7673096, "tokens/train_per_sec_per_gpu": 10.19, "tokens/trainable": 107679} +{"epoch": 0.9296875, "grad_norm": 0.00357458321377635, "learning_rate": 6.408136351831592e-05, "loss": 5.360829891287722e-05, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00005, "step": 238, "tokens/total": 7705344, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 108119} +{"epoch": 0.93359375, "grad_norm": 0.26897430419921875, "learning_rate": 6.379685834195036e-05, "loss": 0.0003528531815391034, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00035, "step": 239, "tokens/total": 7740240, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 108626} +{"epoch": 0.9375, "grad_norm": 0.04304061084985733, "learning_rate": 6.351198709240186e-05, "loss": 0.00041929419967345893, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00042, "step": 240, "tokens/total": 7775392, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 109079} +{"epoch": 0.94140625, "grad_norm": 0.24056600034236908, "learning_rate": 6.32267616243259e-05, "loss": 0.003152453573420644, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00316, "step": 241, "tokens/total": 7807296, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 109542} +{"epoch": 0.9453125, "grad_norm": 0.40210384130477905, "learning_rate": 6.294119380711849e-05, "loss": 0.0012159901671111584, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00122, "step": 242, "tokens/total": 7838200, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 109969} +{"epoch": 0.94921875, "grad_norm": 0.8446518778800964, "learning_rate": 6.265529552442209e-05, "loss": 0.004088847432285547, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.0041, "step": 243, "tokens/total": 7870936, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 110431} +{"epoch": 0.953125, "grad_norm": 0.04028213024139404, "learning_rate": 6.236907867363127e-05, "loss": 0.00029542361153289676, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.0003, "step": 244, "tokens/total": 7902512, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 110877} +{"epoch": 0.95703125, "grad_norm": 0.8272114396095276, "learning_rate": 6.208255516539749e-05, "loss": 0.016723347827792168, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.01686, "step": 245, "tokens/total": 7933224, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 111350} +{"epoch": 0.9609375, "grad_norm": 1.8248883485794067, "learning_rate": 6.179573692313344e-05, "loss": 0.007392487488687038, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00742, "step": 246, "tokens/total": 7968456, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 111798} +{"epoch": 0.96484375, "grad_norm": 0.8818820118904114, "learning_rate": 6.150863588251694e-05, "loss": 0.007478242740035057, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00751, "step": 247, "tokens/total": 8003592, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 112264} +{"epoch": 0.96875, "grad_norm": 0.8696849942207336, "learning_rate": 6.122126399099419e-05, "loss": 0.016295216977596283, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.01643, "step": 248, "tokens/total": 8040048, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 112772} +{"epoch": 0.97265625, "grad_norm": 0.2149159014225006, "learning_rate": 6.0933633207282615e-05, "loss": 0.00154645717702806, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00155, "step": 249, "tokens/total": 8074000, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 113278} +{"epoch": 0.9765625, "grad_norm": 0.34404823184013367, "learning_rate": 6.064575550087316e-05, "loss": 0.005394318141043186, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00541, "step": 250, "tokens/total": 8108304, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 113716} +{"epoch": 0.98046875, "grad_norm": 0.16743260622024536, "learning_rate": 6.0357642851532245e-05, "loss": 0.0024694553576409817, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00247, "step": 251, "tokens/total": 8143464, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 114207} +{"epoch": 0.984375, "grad_norm": 0.5641239285469055, "learning_rate": 6.0069307248803294e-05, "loss": 0.006659465841948986, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00668, "step": 252, "tokens/total": 8175576, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 114628} +{"epoch": 0.98828125, "grad_norm": 0.5832222700119019, "learning_rate": 5.9780760691507635e-05, "loss": 0.005088525824248791, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.0051, "step": 253, "tokens/total": 8205528, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 115060} +{"epoch": 0.9921875, "grad_norm": 0.0808861032128334, "learning_rate": 5.9492015187245334e-05, "loss": 0.0007375560817308724, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00074, "step": 254, "tokens/total": 8239424, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 115514} +{"epoch": 0.99609375, "grad_norm": 1.2718390226364136, "learning_rate": 5.920308275189541e-05, "loss": 0.004771331325173378, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00478, "step": 255, "tokens/total": 8270392, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 115915} +{"epoch": 1.0, "grad_norm": 0.05293723940849304, "learning_rate": 5.8913975409115874e-05, "loss": 0.0009089240338653326, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00091, "step": 256, "tokens/total": 8302096, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 116378} +{"epoch": 1.00390625, "grad_norm": 0.296392023563385, "learning_rate": 5.8624705189843395e-05, "loss": 0.006176782306283712, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.0062, "step": 257, "tokens/total": 8337064, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 116865} +{"epoch": 1.0078125, "grad_norm": 0.25336310267448425, "learning_rate": 5.833528413179249e-05, "loss": 0.0026234586257487535, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00263, "step": 258, "tokens/total": 8372840, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 117363} +{"epoch": 1.01171875, "grad_norm": 0.18495412170886993, "learning_rate": 5.80457242789548e-05, "loss": 0.002324606291949749, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00233, "step": 259, "tokens/total": 8409728, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 117860} +{"epoch": 1.015625, "grad_norm": 0.8740093111991882, "learning_rate": 5.77560376810977e-05, "loss": 0.013726860284805298, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.01382, "step": 260, "tokens/total": 8442296, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 118324} +{"epoch": 1.01953125, "grad_norm": 0.05275081843137741, "learning_rate": 5.7466236393263005e-05, "loss": 0.000749706756323576, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00075, "step": 261, "tokens/total": 8474568, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 118770} +{"epoch": 1.0234375, "grad_norm": 0.5184189677238464, "learning_rate": 5.717633247526522e-05, "loss": 0.0058487518690526485, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00587, "step": 262, "tokens/total": 8505520, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 119206} +{"epoch": 1.02734375, "grad_norm": 0.13834092020988464, "learning_rate": 5.688633799118971e-05, "loss": 0.0014131173957139254, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00141, "step": 263, "tokens/total": 8535064, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 119610} +{"epoch": 1.03125, "grad_norm": 0.07426716387271881, "learning_rate": 5.659626500889066e-05, "loss": 0.0007126101991161704, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00071, "step": 264, "tokens/total": 8563600, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 120080} +{"epoch": 1.03515625, "grad_norm": 0.09398157149553299, "learning_rate": 5.6306125599488905e-05, "loss": 0.0012527642538771033, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00125, "step": 265, "tokens/total": 8592648, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 120519} +{"epoch": 1.0390625, "grad_norm": 0.03612591326236725, "learning_rate": 5.601593183686955e-05, "loss": 0.0006109250243753195, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00061, "step": 266, "tokens/total": 8627328, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 120995} +{"epoch": 1.04296875, "grad_norm": 0.02496417798101902, "learning_rate": 5.572569579717961e-05, "loss": 0.0005195800331421196, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00052, "step": 267, "tokens/total": 8659760, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 121459} +{"epoch": 1.046875, "grad_norm": 0.05653325468301773, "learning_rate": 5.543542955832538e-05, "loss": 0.00037323220749385655, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00037, "step": 268, "tokens/total": 8696968, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 121902} +{"epoch": 1.05078125, "grad_norm": 0.33634644746780396, "learning_rate": 5.514514519946986e-05, "loss": 0.006432520225644112, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00645, "step": 269, "tokens/total": 8735392, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 122399} +{"epoch": 1.0546875, "grad_norm": 0.30202996730804443, "learning_rate": 5.485485480053015e-05, "loss": 0.0023846998810768127, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00239, "step": 270, "tokens/total": 8769840, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 122804} +{"epoch": 1.05859375, "grad_norm": 0.017934799194335938, "learning_rate": 5.4564570441674645e-05, "loss": 0.0002452124026603997, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00025, "step": 271, "tokens/total": 8800648, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 123244} +{"epoch": 1.0625, "grad_norm": 0.017715200781822205, "learning_rate": 5.42743042028204e-05, "loss": 0.00026809098199009895, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00027, "step": 272, "tokens/total": 8832480, "tokens/train_per_sec_per_gpu": 6.86, "tokens/trainable": 123683} +{"epoch": 1.06640625, "grad_norm": 0.4231114685535431, "learning_rate": 5.3984068163130464e-05, "loss": 0.0014898061053827405, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00149, "step": 273, "tokens/total": 8864840, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 124153} +{"epoch": 1.0703125, "grad_norm": 0.17331132292747498, "learning_rate": 5.369387440051111e-05, "loss": 0.0008813315653242171, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00088, "step": 274, "tokens/total": 8897272, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 124616} +{"epoch": 1.07421875, "grad_norm": 0.017663750797510147, "learning_rate": 5.340373499110935e-05, "loss": 0.00018265507242176682, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00018, "step": 275, "tokens/total": 8930688, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 125109} +{"epoch": 1.078125, "grad_norm": 0.007485608570277691, "learning_rate": 5.3113662008810304e-05, "loss": 7.083182572387159e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00007, "step": 276, "tokens/total": 8967216, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 125591} +{"epoch": 1.08203125, "grad_norm": 0.10113867372274399, "learning_rate": 5.282366752473479e-05, "loss": 0.000672101799864322, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00067, "step": 277, "tokens/total": 9001368, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 126065} +{"epoch": 1.0859375, "grad_norm": 0.0039147608913481236, "learning_rate": 5.2533763606737005e-05, "loss": 6.912929529789835e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00007, "step": 278, "tokens/total": 9036808, "tokens/train_per_sec_per_gpu": 6.94, "tokens/trainable": 126480} +{"epoch": 1.08984375, "grad_norm": 0.04177643731236458, "learning_rate": 5.224396231890232e-05, "loss": 0.00032682737219147384, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00033, "step": 279, "tokens/total": 9067840, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 126929} +{"epoch": 1.09375, "grad_norm": 0.23373356461524963, "learning_rate": 5.195427572104522e-05, "loss": 0.004324289970099926, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00433, "step": 280, "tokens/total": 9098936, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 127366} +{"epoch": 1.09765625, "grad_norm": 0.20071950554847717, "learning_rate": 5.166471586820751e-05, "loss": 0.0009351305780000985, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00094, "step": 281, "tokens/total": 9133264, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 127831} +{"epoch": 1.1015625, "grad_norm": 0.1570623368024826, "learning_rate": 5.1375294810156615e-05, "loss": 0.0006069710943847895, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00061, "step": 282, "tokens/total": 9165008, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 128278} +{"epoch": 1.10546875, "grad_norm": 0.26303714513778687, "learning_rate": 5.1086024590884144e-05, "loss": 0.0013141232775524259, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00131, "step": 283, "tokens/total": 9197952, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 128727} +{"epoch": 1.109375, "grad_norm": 0.2119615077972412, "learning_rate": 5.079691724810461e-05, "loss": 0.0019583520479500294, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00196, "step": 284, "tokens/total": 9232576, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 129162} +{"epoch": 1.11328125, "grad_norm": 0.007372864987701178, "learning_rate": 5.0507984812754684e-05, "loss": 8.027484000194818e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00008, "step": 285, "tokens/total": 9264568, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 129659} +{"epoch": 1.1171875, "grad_norm": 0.727709174156189, "learning_rate": 5.021923930849237e-05, "loss": 0.005353633780032396, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00537, "step": 286, "tokens/total": 9296544, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 130145} +{"epoch": 1.12109375, "grad_norm": 0.003009217558428645, "learning_rate": 4.99306927511967e-05, "loss": 1.0049791853816714e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00001, "step": 287, "tokens/total": 9327936, "tokens/train_per_sec_per_gpu": 10.46, "tokens/trainable": 130585} +{"epoch": 1.125, "grad_norm": 0.018195712938904762, "learning_rate": 4.964235714846775e-05, "loss": 0.00013086560647934675, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00013, "step": 288, "tokens/total": 9357984, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 131005} +{"epoch": 1.12890625, "grad_norm": 0.0024116577114909887, "learning_rate": 4.9354244499126866e-05, "loss": 2.0540072000585496e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00002, "step": 289, "tokens/total": 9388664, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 131471} +{"epoch": 1.1328125, "grad_norm": 0.2686386704444885, "learning_rate": 4.90663667927174e-05, "loss": 0.0010491784196346998, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00105, "step": 290, "tokens/total": 9420504, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 131920} +{"epoch": 1.13671875, "grad_norm": 0.2290729582309723, "learning_rate": 4.877873600900581e-05, "loss": 0.0016215462237596512, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00162, "step": 291, "tokens/total": 9452664, "tokens/train_per_sec_per_gpu": 6.34, "tokens/trainable": 132379} +{"epoch": 1.140625, "grad_norm": 0.04138621687889099, "learning_rate": 4.849136411748306e-05, "loss": 0.00020099722314625978, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.0002, "step": 292, "tokens/total": 9488896, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 132851} +{"epoch": 1.14453125, "grad_norm": 30.850309371948242, "learning_rate": 4.8204263076866574e-05, "loss": 0.017539197579026222, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01769, "step": 293, "tokens/total": 9522112, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 133351} +{"epoch": 1.1484375, "grad_norm": 0.008210251107811928, "learning_rate": 4.791744483460251e-05, "loss": 5.4675943829352036e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00005, "step": 294, "tokens/total": 9555632, "tokens/train_per_sec_per_gpu": 6.62, "tokens/trainable": 133784} +{"epoch": 1.15234375, "grad_norm": 0.0015149795217439532, "learning_rate": 4.7630921326368736e-05, "loss": 1.4504123100778088e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00001, "step": 295, "tokens/total": 9590904, "tokens/train_per_sec_per_gpu": 6.45, "tokens/trainable": 134209} +{"epoch": 1.15625, "grad_norm": 0.3025225102901459, "learning_rate": 4.7344704475577916e-05, "loss": 0.0009720124071463943, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00097, "step": 296, "tokens/total": 9620288, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 134645} +{"epoch": 1.16015625, "grad_norm": 0.01928967982530594, "learning_rate": 4.705880619288153e-05, "loss": 0.00012335175415500998, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00012, "step": 297, "tokens/total": 9651984, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 135047} +{"epoch": 1.1640625, "grad_norm": 0.027275118976831436, "learning_rate": 4.677323837567412e-05, "loss": 0.00017368615954183042, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00017, "step": 298, "tokens/total": 9681368, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 135474} +{"epoch": 1.16796875, "grad_norm": 0.5999098420143127, "learning_rate": 4.6488012907598146e-05, "loss": 0.008276823908090591, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00831, "step": 299, "tokens/total": 9712872, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 135948} +{"epoch": 1.171875, "grad_norm": 0.7358774542808533, "learning_rate": 4.620314165804964e-05, "loss": 0.003880084492266178, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00389, "step": 300, "tokens/total": 9745184, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 136421} +{"epoch": 1.17578125, "grad_norm": 0.5047712922096252, "learning_rate": 4.591863648168407e-05, "loss": 0.004210661631077528, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00422, "step": 301, "tokens/total": 9778928, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 136886} +{"epoch": 1.1796875, "grad_norm": 0.7350596785545349, "learning_rate": 4.5634509217923135e-05, "loss": 0.013722526840865612, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.01382, "step": 302, "tokens/total": 9810752, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 137321} +{"epoch": 1.18359375, "grad_norm": 0.7234786152839661, "learning_rate": 4.535077169046201e-05, "loss": 0.007589112035930157, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00762, "step": 303, "tokens/total": 9843672, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 137758} +{"epoch": 1.1875, "grad_norm": 0.35101476311683655, "learning_rate": 4.506743570677743e-05, "loss": 0.0019613674376159906, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00196, "step": 304, "tokens/total": 9873752, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 138190} +{"epoch": 1.19140625, "grad_norm": 0.010966155678033829, "learning_rate": 4.478451305763618e-05, "loss": 7.249194459291175e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00007, "step": 305, "tokens/total": 9906272, "tokens/train_per_sec_per_gpu": 9.88, "tokens/trainable": 138650} +{"epoch": 1.1953125, "grad_norm": 0.0065241605043411255, "learning_rate": 4.450201551660454e-05, "loss": 4.502440424403176e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00005, "step": 306, "tokens/total": 9936296, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 139082} +{"epoch": 1.19921875, "grad_norm": 0.10654430091381073, "learning_rate": 4.4219954839558276e-05, "loss": 0.0005409215809777379, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00054, "step": 307, "tokens/total": 9967720, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 139523} +{"epoch": 1.203125, "grad_norm": 0.2104233354330063, "learning_rate": 4.393834276419352e-05, "loss": 0.0022159942891448736, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.00222, "step": 308, "tokens/total": 9997176, "tokens/train_per_sec_per_gpu": 10.87, "tokens/trainable": 140010} +{"epoch": 1.20703125, "grad_norm": 0.04036187380552292, "learning_rate": 4.36571910095382e-05, "loss": 0.000363894592737779, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00036, "step": 309, "tokens/total": 10031872, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 140454} +{"epoch": 1.2109375, "grad_norm": 0.7728093266487122, "learning_rate": 4.337651127546448e-05, "loss": 0.004919679835438728, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00493, "step": 310, "tokens/total": 10065240, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 140905} +{"epoch": 1.21484375, "grad_norm": 0.193764790892601, "learning_rate": 4.3096315242201736e-05, "loss": 0.002252563601359725, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00226, "step": 311, "tokens/total": 10096640, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 141339} +{"epoch": 1.21875, "grad_norm": 0.005182007327675819, "learning_rate": 4.2816614569850635e-05, "loss": 7.501310028601438e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00008, "step": 312, "tokens/total": 10130192, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 141778} +{"epoch": 1.22265625, "grad_norm": 0.31879401206970215, "learning_rate": 4.2537420897897864e-05, "loss": 0.008607027120888233, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00864, "step": 313, "tokens/total": 10164248, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 142255} +{"epoch": 1.2265625, "grad_norm": 0.11906725913286209, "learning_rate": 4.225874584473174e-05, "loss": 0.001628158031962812, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00163, "step": 314, "tokens/total": 10194664, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 142693} +{"epoch": 1.23046875, "grad_norm": 0.23018646240234375, "learning_rate": 4.19806010071587e-05, "loss": 0.0016023520147427917, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.0016, "step": 315, "tokens/total": 10227768, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 143157} +{"epoch": 1.234375, "grad_norm": 0.2708475589752197, "learning_rate": 4.170299795992081e-05, "loss": 0.0034634843468666077, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00347, "step": 316, "tokens/total": 10260464, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 143603} +{"epoch": 1.23828125, "grad_norm": 0.1508363038301468, "learning_rate": 4.142594825521398e-05, "loss": 0.0013866880908608437, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00139, "step": 317, "tokens/total": 10297528, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 144084} +{"epoch": 1.2421875, "grad_norm": 0.7492601275444031, "learning_rate": 4.114946342220728e-05, "loss": 0.0059348284266889095, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00595, "step": 318, "tokens/total": 10331448, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 144532} +{"epoch": 1.24609375, "grad_norm": 0.2235262542963028, "learning_rate": 4.087355496656321e-05, "loss": 0.0023946918081492186, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.0024, "step": 319, "tokens/total": 10363152, "tokens/train_per_sec_per_gpu": 10.23, "tokens/trainable": 145004} +{"epoch": 1.25, "grad_norm": 0.3566429615020752, "learning_rate": 4.05982343699588e-05, "loss": 0.009839482605457306, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00989, "step": 320, "tokens/total": 10397832, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 145498} +{"epoch": 1.25390625, "grad_norm": 0.0776422768831253, "learning_rate": 4.0323513089607876e-05, "loss": 0.0006294260965660214, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00063, "step": 321, "tokens/total": 10431416, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 145965} +{"epoch": 1.2578125, "grad_norm": 0.045666273683309555, "learning_rate": 4.004940255778431e-05, "loss": 0.0003902872558683157, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00039, "step": 322, "tokens/total": 10467384, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 146409} +{"epoch": 1.26171875, "grad_norm": 0.0141483498737216, "learning_rate": 3.977591418134619e-05, "loss": 0.0002418474032310769, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00024, "step": 323, "tokens/total": 10500792, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 146849} +{"epoch": 1.265625, "grad_norm": 0.19516953825950623, "learning_rate": 3.95030593412612e-05, "loss": 0.0008903526468202472, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00089, "step": 324, "tokens/total": 10532592, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 147272} +{"epoch": 1.26953125, "grad_norm": 0.3618069887161255, "learning_rate": 3.923084939213296e-05, "loss": 0.004495879635214806, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00451, "step": 325, "tokens/total": 10562392, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 147694} +{"epoch": 1.2734375, "grad_norm": 0.017977630719542503, "learning_rate": 3.895929566172861e-05, "loss": 0.00026827596593648195, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00027, "step": 326, "tokens/total": 10595416, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 148158} +{"epoch": 1.27734375, "grad_norm": 0.0707300528883934, "learning_rate": 3.868840945050728e-05, "loss": 0.0006486140773631632, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00065, "step": 327, "tokens/total": 10630504, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 148590} +{"epoch": 1.28125, "grad_norm": 0.013556229881942272, "learning_rate": 3.841820203114995e-05, "loss": 0.00017305831715930253, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00017, "step": 328, "tokens/total": 10661632, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 149037} +{"epoch": 1.28515625, "grad_norm": 0.1264519840478897, "learning_rate": 3.814868464809027e-05, "loss": 0.0017875880002975464, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00179, "step": 329, "tokens/total": 10691544, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 149439} +{"epoch": 1.2890625, "grad_norm": 0.26211345195770264, "learning_rate": 3.787986851704667e-05, "loss": 0.002543138340115547, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00255, "step": 330, "tokens/total": 10724768, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 149869} +{"epoch": 1.29296875, "grad_norm": 0.2527926564216614, "learning_rate": 3.7611764824555654e-05, "loss": 0.0006484196637757123, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00065, "step": 331, "tokens/total": 10756008, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 150343} +{"epoch": 1.296875, "grad_norm": 0.2982660233974457, "learning_rate": 3.734438472750619e-05, "loss": 0.0026124627329409122, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00262, "step": 332, "tokens/total": 10793152, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 150814} +{"epoch": 1.30078125, "grad_norm": 0.006268317811191082, "learning_rate": 3.707773935267552e-05, "loss": 7.182340777944773e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00007, "step": 333, "tokens/total": 10827624, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 151300} +{"epoch": 1.3046875, "grad_norm": 0.14606231451034546, "learning_rate": 3.68118397962661e-05, "loss": 0.0008110678754746914, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00081, "step": 334, "tokens/total": 10859712, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 151708} +{"epoch": 1.30859375, "grad_norm": 0.28271961212158203, "learning_rate": 3.654669712344384e-05, "loss": 0.0007122554816305637, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00071, "step": 335, "tokens/total": 10888792, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 152164} +{"epoch": 1.3125, "grad_norm": 0.0061697568744421005, "learning_rate": 3.628232236787763e-05, "loss": 6.602222856599838e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00007, "step": 336, "tokens/total": 10922968, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 152603} +{"epoch": 1.31640625, "grad_norm": 0.07710785418748856, "learning_rate": 3.6018726531280144e-05, "loss": 0.00042295752791687846, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00042, "step": 337, "tokens/total": 10958000, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 153094} +{"epoch": 1.3203125, "grad_norm": 0.4469854533672333, "learning_rate": 3.575592058295017e-05, "loss": 0.007266640197485685, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00729, "step": 338, "tokens/total": 10989872, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 153573} +{"epoch": 1.32421875, "grad_norm": 0.001143199042417109, "learning_rate": 3.549391545931585e-05, "loss": 1.5947412975947373e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00002, "step": 339, "tokens/total": 11020784, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 154033} +{"epoch": 1.328125, "grad_norm": 0.012896324507892132, "learning_rate": 3.5232722063479914e-05, "loss": 0.00011775091843446717, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00012, "step": 340, "tokens/total": 11053240, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 154482} +{"epoch": 1.33203125, "grad_norm": 0.03497753664851189, "learning_rate": 3.49723512647657e-05, "loss": 0.00027394152130000293, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00027, "step": 341, "tokens/total": 11089040, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 154982} +{"epoch": 1.3359375, "grad_norm": 0.6943045258522034, "learning_rate": 3.471281389826491e-05, "loss": 0.00013032951392233372, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00013, "step": 342, "tokens/total": 11119672, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 155387} +{"epoch": 1.33984375, "grad_norm": 0.12573108077049255, "learning_rate": 3.4454120764386764e-05, "loss": 0.0019628230948001146, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00196, "step": 343, "tokens/total": 11155040, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 155842} +{"epoch": 1.34375, "grad_norm": 0.0026098520029336214, "learning_rate": 3.4196282628408526e-05, "loss": 3.654843749245629e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00004, "step": 344, "tokens/total": 11185952, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 156287} +{"epoch": 1.34765625, "grad_norm": 0.15116526186466217, "learning_rate": 3.3939310220027456e-05, "loss": 0.0013927179388701916, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00139, "step": 345, "tokens/total": 11219272, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 156782} +{"epoch": 1.3515625, "grad_norm": 0.0013979289215058088, "learning_rate": 3.3683214232914404e-05, "loss": 2.13688599615125e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00002, "step": 346, "tokens/total": 11253424, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 157255} +{"epoch": 1.35546875, "grad_norm": 0.13907749950885773, "learning_rate": 3.342800532426873e-05, "loss": 0.00039339307113550603, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00039, "step": 347, "tokens/total": 11286216, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 157721} +{"epoch": 1.359375, "grad_norm": 0.057782381772994995, "learning_rate": 3.317369411437484e-05, "loss": 0.0005380093352869153, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00054, "step": 348, "tokens/total": 11319480, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 158189} +{"epoch": 1.36328125, "grad_norm": 0.1325269192457199, "learning_rate": 3.292029118616024e-05, "loss": 0.00039234227733686566, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00039, "step": 349, "tokens/total": 11350472, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 158612} +{"epoch": 1.3671875, "grad_norm": 0.2944362163543701, "learning_rate": 3.266780708475511e-05, "loss": 0.006010440178215504, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00603, "step": 350, "tokens/total": 11383088, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 159090} +{"epoch": 1.37109375, "grad_norm": 0.2328377366065979, "learning_rate": 3.241625231705354e-05, "loss": 0.0009757736697793007, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00098, "step": 351, "tokens/total": 11418392, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 159549} +{"epoch": 1.375, "grad_norm": 0.0014445068081840873, "learning_rate": 3.216563735127618e-05, "loss": 2.2987032934906892e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00002, "step": 352, "tokens/total": 11449784, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 160026} +{"epoch": 1.37890625, "grad_norm": 0.3266206979751587, "learning_rate": 3.191597261653475e-05, "loss": 0.004016886930912733, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00402, "step": 353, "tokens/total": 11484240, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 160487} +{"epoch": 1.3828125, "grad_norm": 0.06324172019958496, "learning_rate": 3.166726850239794e-05, "loss": 0.00038323694025166333, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00038, "step": 354, "tokens/total": 11513360, "tokens/train_per_sec_per_gpu": 11.09, "tokens/trainable": 160935} +{"epoch": 1.38671875, "grad_norm": 0.5920830965042114, "learning_rate": 3.141953535845912e-05, "loss": 0.0005591318476945162, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00056, "step": 355, "tokens/total": 11543112, "tokens/train_per_sec_per_gpu": 10.33, "tokens/trainable": 161385} +{"epoch": 1.390625, "grad_norm": 0.006380106322467327, "learning_rate": 3.11727834939056e-05, "loss": 9.489757940173149e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00009, "step": 356, "tokens/total": 11573464, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 161849} +{"epoch": 1.39453125, "grad_norm": 0.1353534758090973, "learning_rate": 3.092702317708967e-05, "loss": 0.0019280733540654182, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00193, "step": 357, "tokens/total": 11604896, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 162306} +{"epoch": 1.3984375, "grad_norm": 0.021009523421525955, "learning_rate": 3.0682264635101276e-05, "loss": 0.0002117530384566635, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00021, "step": 358, "tokens/total": 11636368, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 162759} +{"epoch": 1.40234375, "grad_norm": 0.24694114923477173, "learning_rate": 3.0438518053342407e-05, "loss": 0.0024408053141087294, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00244, "step": 359, "tokens/total": 11670216, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 163230} +{"epoch": 1.40625, "grad_norm": 0.07537376880645752, "learning_rate": 3.0195793575103266e-05, "loss": 0.0008646540809422731, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00087, "step": 360, "tokens/total": 11707816, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 163694} +{"epoch": 1.41015625, "grad_norm": 0.2221444547176361, "learning_rate": 2.9954101301140146e-05, "loss": 0.002378304721787572, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00238, "step": 361, "tokens/total": 11742584, "tokens/train_per_sec_per_gpu": 6.96, "tokens/trainable": 164144} +{"epoch": 1.4140625, "grad_norm": 0.34937596321105957, "learning_rate": 2.9713451289255123e-05, "loss": 0.0018395914230495691, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00184, "step": 362, "tokens/total": 11770384, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 164585} +{"epoch": 1.41796875, "grad_norm": 0.0016059677582234144, "learning_rate": 2.9473853553877484e-05, "loss": 1.544625047245063e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00002, "step": 363, "tokens/total": 11803208, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 165059} +{"epoch": 1.421875, "grad_norm": 0.054400257766246796, "learning_rate": 2.9235318065647e-05, "loss": 0.00016826610954012722, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00017, "step": 364, "tokens/total": 11840480, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 165519} +{"epoch": 1.42578125, "grad_norm": 0.004019748419523239, "learning_rate": 2.8997854750998964e-05, "loss": 4.32889792136848e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00004, "step": 365, "tokens/total": 11875544, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 165984} +{"epoch": 1.4296875, "grad_norm": 0.002781103365123272, "learning_rate": 2.8761473491751258e-05, "loss": 1.945624899235554e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00002, "step": 366, "tokens/total": 11907656, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 166419} +{"epoch": 1.43359375, "grad_norm": 0.03632962703704834, "learning_rate": 2.8526184124692883e-05, "loss": 0.00018709752475842834, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00019, "step": 367, "tokens/total": 11941904, "tokens/train_per_sec_per_gpu": 6.46, "tokens/trainable": 166857} +{"epoch": 1.4375, "grad_norm": 1.5764309167861938, "learning_rate": 2.829199644117484e-05, "loss": 0.032166317105293274, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.03269, "step": 368, "tokens/total": 11973624, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 167267} +{"epoch": 1.44140625, "grad_norm": 0.057281166315078735, "learning_rate": 2.8058920186702553e-05, "loss": 0.00044676160905510187, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00045, "step": 369, "tokens/total": 12010360, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 167740} +{"epoch": 1.4453125, "grad_norm": 0.0014105014270171523, "learning_rate": 2.782696506053033e-05, "loss": 1.8521224774303846e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00002, "step": 370, "tokens/total": 12044544, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 168172} +{"epoch": 1.44921875, "grad_norm": 0.10535351186990738, "learning_rate": 2.7596140715257824e-05, "loss": 0.000541514134965837, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00054, "step": 371, "tokens/total": 12075792, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 168637} +{"epoch": 1.453125, "grad_norm": 0.0020804640371352434, "learning_rate": 2.7366456756428184e-05, "loss": 2.9079084924887866e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00003, "step": 372, "tokens/total": 12108088, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 169094} +{"epoch": 1.45703125, "grad_norm": 0.043212663382291794, "learning_rate": 2.7137922742128486e-05, "loss": 0.0003916345885954797, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00039, "step": 373, "tokens/total": 12140888, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 169593} +{"epoch": 1.4609375, "grad_norm": 0.006111972499638796, "learning_rate": 2.691054818259188e-05, "loss": 6.374213262461126e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00006, "step": 374, "tokens/total": 12174560, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 170066} +{"epoch": 1.46484375, "grad_norm": 0.26605546474456787, "learning_rate": 2.6684342539801933e-05, "loss": 0.0025421008467674255, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00255, "step": 375, "tokens/total": 12204848, "tokens/train_per_sec_per_gpu": 10.04, "tokens/trainable": 170518} +{"epoch": 1.46875, "grad_norm": 0.12353689968585968, "learning_rate": 2.645931522709877e-05, "loss": 0.0018983000190928578, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.0019, "step": 376, "tokens/total": 12238472, "tokens/train_per_sec_per_gpu": 10.84, "tokens/trainable": 170998} +{"epoch": 1.47265625, "grad_norm": 0.015089573338627815, "learning_rate": 2.6235475608787365e-05, "loss": 0.00017767242388799787, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00018, "step": 377, "tokens/total": 12271024, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 171459} +{"epoch": 1.4765625, "grad_norm": 0.07860875874757767, "learning_rate": 2.6012832999747916e-05, "loss": 0.0005231890827417374, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00052, "step": 378, "tokens/total": 12301312, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 171905} +{"epoch": 1.48046875, "grad_norm": 0.1818556934595108, "learning_rate": 2.579139666504821e-05, "loss": 0.002214832929894328, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00222, "step": 379, "tokens/total": 12337072, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 172351} +{"epoch": 1.484375, "grad_norm": 0.04338269680738449, "learning_rate": 2.557117581955798e-05, "loss": 0.00031715453951619565, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00032, "step": 380, "tokens/total": 12368000, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 172817} +{"epoch": 1.48828125, "grad_norm": 0.48399725556373596, "learning_rate": 2.5352179627565532e-05, "loss": 0.01853015646338463, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.0187, "step": 381, "tokens/total": 12399880, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 173258} +{"epoch": 1.4921875, "grad_norm": 0.05254250392317772, "learning_rate": 2.5134417202396277e-05, "loss": 0.00022321587312035263, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00022, "step": 382, "tokens/total": 12432288, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 173665} +{"epoch": 1.49609375, "grad_norm": 0.006623541470617056, "learning_rate": 2.491789760603361e-05, "loss": 0.0001193653151858598, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00012, "step": 383, "tokens/total": 12465592, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 174095} +{"epoch": 1.5, "grad_norm": 0.12483753263950348, "learning_rate": 2.4702629848741764e-05, "loss": 0.0010065680835396051, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00101, "step": 384, "tokens/total": 12498200, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 174550} +{"epoch": 1.50390625, "grad_norm": 0.2505822479724884, "learning_rate": 2.4488622888690785e-05, "loss": 0.002298625884577632, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.0023, "step": 385, "tokens/total": 12530280, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 174990} +{"epoch": 1.5078125, "grad_norm": 0.0766463577747345, "learning_rate": 2.427588563158384e-05, "loss": 0.0007713482482358813, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00077, "step": 386, "tokens/total": 12563360, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 175476} +{"epoch": 1.51171875, "grad_norm": 0.16685688495635986, "learning_rate": 2.406442693028651e-05, "loss": 0.0027177531737834215, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00272, "step": 387, "tokens/total": 12593032, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 175897} +{"epoch": 1.515625, "grad_norm": 0.0035879909992218018, "learning_rate": 2.3854255584458547e-05, "loss": 5.9353100368753076e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00006, "step": 388, "tokens/total": 12628272, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 176324} +{"epoch": 1.51953125, "grad_norm": 0.056232381612062454, "learning_rate": 2.3645380340187508e-05, "loss": 0.0001411235862178728, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00014, "step": 389, "tokens/total": 12661368, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 176805} +{"epoch": 1.5234375, "grad_norm": 0.030527355149388313, "learning_rate": 2.3437809889624914e-05, "loss": 0.00032246712362393737, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.77, "memory/max_allocated (GiB)": 63.77, "ppl": 1.00032, "step": 390, "tokens/total": 12691168, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 177256} +{"epoch": 1.52734375, "grad_norm": 0.010087860748171806, "learning_rate": 2.3231552870624487e-05, "loss": 0.00015876472753006965, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00016, "step": 391, "tokens/total": 12721264, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 177682} +{"epoch": 1.53125, "grad_norm": 0.011808055453002453, "learning_rate": 2.3026617866382657e-05, "loss": 9.661550575401634e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.0001, "step": 392, "tokens/total": 12753640, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 178179} +{"epoch": 1.53515625, "grad_norm": 0.017954209819436073, "learning_rate": 2.2823013405081507e-05, "loss": 0.00012591791164595634, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00013, "step": 393, "tokens/total": 12787056, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 178676} +{"epoch": 1.5390625, "grad_norm": 0.011038695462048054, "learning_rate": 2.2620747959533722e-05, "loss": 0.00012993215932510793, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00013, "step": 394, "tokens/total": 12819312, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 179173} +{"epoch": 1.54296875, "grad_norm": 0.004947492387145758, "learning_rate": 2.2419829946830123e-05, "loss": 4.727220584754832e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00005, "step": 395, "tokens/total": 12853376, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 179624} +{"epoch": 1.546875, "grad_norm": 0.022992579266428947, "learning_rate": 2.2220267727989325e-05, "loss": 0.0002096962125506252, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00021, "step": 396, "tokens/total": 12882960, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 180078} +{"epoch": 1.55078125, "grad_norm": 0.03816591575741768, "learning_rate": 2.202206960760984e-05, "loss": 0.00042978551937267184, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00043, "step": 397, "tokens/total": 12917016, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 180585} +{"epoch": 1.5546875, "grad_norm": 0.04290127009153366, "learning_rate": 2.182524383352446e-05, "loss": 0.0005912703345529735, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00059, "step": 398, "tokens/total": 12950056, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 181039} +{"epoch": 1.55859375, "grad_norm": 0.03839961811900139, "learning_rate": 2.1629798596457056e-05, "loss": 0.00043626222759485245, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00044, "step": 399, "tokens/total": 12981728, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 181489} +{"epoch": 1.5625, "grad_norm": 0.4747318625450134, "learning_rate": 2.1435742029681725e-05, "loss": 0.00649057375267148, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00651, "step": 400, "tokens/total": 13011464, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 181906} +{"epoch": 1.56640625, "grad_norm": 0.003130875062197447, "learning_rate": 2.124308220868431e-05, "loss": 3.4062661143252626e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00003, "step": 401, "tokens/total": 13044432, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 182361} +{"epoch": 1.5703125, "grad_norm": 0.04355934262275696, "learning_rate": 2.105182715082638e-05, "loss": 0.00030435819644480944, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.0003, "step": 402, "tokens/total": 13075608, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 182838} +{"epoch": 1.57421875, "grad_norm": 0.1952192336320877, "learning_rate": 2.0861984815011552e-05, "loss": 0.003136272542178631, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00314, "step": 403, "tokens/total": 13107352, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 183269} +{"epoch": 1.578125, "grad_norm": 0.024943260475993156, "learning_rate": 2.0673563101354323e-05, "loss": 0.0004405776271596551, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00044, "step": 404, "tokens/total": 13140736, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 183699} +{"epoch": 1.58203125, "grad_norm": 0.10554702579975128, "learning_rate": 2.0486569850851317e-05, "loss": 0.000631032045930624, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00063, "step": 405, "tokens/total": 13174040, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 184162} +{"epoch": 1.5859375, "grad_norm": 0.005724010057747364, "learning_rate": 2.0301012845054956e-05, "loss": 8.801170042715967e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00009, "step": 406, "tokens/total": 13207232, "tokens/train_per_sec_per_gpu": 6.65, "tokens/trainable": 184576} +{"epoch": 1.58984375, "grad_norm": 0.3142811357975006, "learning_rate": 2.011689980574966e-05, "loss": 0.0022414326667785645, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00224, "step": 407, "tokens/total": 13241160, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 184995} +{"epoch": 1.59375, "grad_norm": 0.023884015157818794, "learning_rate": 1.993423839463052e-05, "loss": 0.0003123147471342236, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00031, "step": 408, "tokens/total": 13273104, "tokens/train_per_sec_per_gpu": 10.5, "tokens/trainable": 185452} +{"epoch": 1.59765625, "grad_norm": 0.01254682894796133, "learning_rate": 1.975303621298445e-05, "loss": 0.00018408562755212188, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00018, "step": 409, "tokens/total": 13308136, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 185904} +{"epoch": 1.6015625, "grad_norm": 0.007542282808572054, "learning_rate": 1.957330080137385e-05, "loss": 7.950256258482113e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00008, "step": 410, "tokens/total": 13337816, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 186355} +{"epoch": 1.60546875, "grad_norm": 0.017269648611545563, "learning_rate": 1.9395039639322864e-05, "loss": 0.0002379427314735949, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00024, "step": 411, "tokens/total": 13372576, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 186831} +{"epoch": 1.609375, "grad_norm": 0.006068653427064419, "learning_rate": 1.9218260145006073e-05, "loss": 3.695105988299474e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00004, "step": 412, "tokens/total": 13406664, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 187317} +{"epoch": 1.61328125, "grad_norm": 0.002160759409889579, "learning_rate": 1.904296967493982e-05, "loss": 3.5889577702619135e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00004, "step": 413, "tokens/total": 13438256, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 187773} +{"epoch": 1.6171875, "grad_norm": 0.010062429122626781, "learning_rate": 1.8869175523676064e-05, "loss": 0.00012876502296421677, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00013, "step": 414, "tokens/total": 13472096, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 188268} +{"epoch": 1.62109375, "grad_norm": 0.07114101201295853, "learning_rate": 1.869688492349885e-05, "loss": 0.0001863790675997734, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00019, "step": 415, "tokens/total": 13504192, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 188720} +{"epoch": 1.625, "grad_norm": 0.007742891553789377, "learning_rate": 1.85261050441233e-05, "loss": 0.00010469413246028125, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.0001, "step": 416, "tokens/total": 13537392, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 189177} +{"epoch": 1.62890625, "grad_norm": 0.09939311444759369, "learning_rate": 1.8356842992397304e-05, "loss": 0.0006217347108758986, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00062, "step": 417, "tokens/total": 13569344, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 189617} +{"epoch": 1.6328125, "grad_norm": 0.013712972402572632, "learning_rate": 1.8189105812005714e-05, "loss": 0.0001419715117663145, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00014, "step": 418, "tokens/total": 13602200, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 190028} +{"epoch": 1.63671875, "grad_norm": 0.0018547115614637733, "learning_rate": 1.802290048317732e-05, "loss": 3.5478107747621834e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00004, "step": 419, "tokens/total": 13632640, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 190445} +{"epoch": 1.640625, "grad_norm": 0.016982486471533775, "learning_rate": 1.785823392239424e-05, "loss": 0.00013236743689049035, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00013, "step": 420, "tokens/total": 13664920, "tokens/train_per_sec_per_gpu": 10.04, "tokens/trainable": 190886} +{"epoch": 1.64453125, "grad_norm": 0.13668350875377655, "learning_rate": 1.7695112982104225e-05, "loss": 0.0002850755408871919, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00029, "step": 421, "tokens/total": 13697144, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 191325} +{"epoch": 1.6484375, "grad_norm": 0.11296498030424118, "learning_rate": 1.7533544450435433e-05, "loss": 0.0008430880261585116, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00084, "step": 422, "tokens/total": 13724496, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 191758} +{"epoch": 1.65234375, "grad_norm": 0.01794513873755932, "learning_rate": 1.7373535050913946e-05, "loss": 7.381912291748449e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00007, "step": 423, "tokens/total": 13757880, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 192195} +{"epoch": 1.65625, "grad_norm": 0.12161166965961456, "learning_rate": 1.721509144218405e-05, "loss": 0.00013663114805240184, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00014, "step": 424, "tokens/total": 13787648, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 192625} +{"epoch": 1.66015625, "grad_norm": 0.024480827152729034, "learning_rate": 1.705822021773101e-05, "loss": 0.0001587279693922028, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00016, "step": 425, "tokens/total": 13819720, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 193111} +{"epoch": 1.6640625, "grad_norm": 0.11978563666343689, "learning_rate": 1.69029279056068e-05, "loss": 0.000669859116896987, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00067, "step": 426, "tokens/total": 13854944, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 193551} +{"epoch": 1.66796875, "grad_norm": 0.0052636731415987015, "learning_rate": 1.6749220968158415e-05, "loss": 1.973404323507566e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00002, "step": 427, "tokens/total": 13887824, "tokens/train_per_sec_per_gpu": 10.43, "tokens/trainable": 194057} +{"epoch": 1.671875, "grad_norm": 0.014907952398061752, "learning_rate": 1.659710580175893e-05, "loss": 0.00015279512444976717, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00015, "step": 428, "tokens/total": 13918192, "tokens/train_per_sec_per_gpu": 10.22, "tokens/trainable": 194556} +{"epoch": 1.67578125, "grad_norm": 0.001115199876949191, "learning_rate": 1.644658873654133e-05, "loss": 2.4040244170464575e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00002, "step": 429, "tokens/total": 13947440, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 194993} +{"epoch": 1.6796875, "grad_norm": 0.0009709961013868451, "learning_rate": 1.629767603613508e-05, "loss": 1.6514683011337183e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00002, "step": 430, "tokens/total": 13974360, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 195393} +{"epoch": 1.68359375, "grad_norm": 0.015268804505467415, "learning_rate": 1.615037389740547e-05, "loss": 0.00011412057938287035, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00011, "step": 431, "tokens/total": 14006592, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 195837} +{"epoch": 1.6875, "grad_norm": 0.007118393667042255, "learning_rate": 1.600468845019576e-05, "loss": 7.706474571023136e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00008, "step": 432, "tokens/total": 14040776, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 196317} +{"epoch": 1.69140625, "grad_norm": 0.0005067043239250779, "learning_rate": 1.5860625757072092e-05, "loss": 9.795399819267914e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00001, "step": 433, "tokens/total": 14073040, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 196765} +{"epoch": 1.6953125, "grad_norm": 0.0038482611998915672, "learning_rate": 1.571819181307116e-05, "loss": 3.70015186490491e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00004, "step": 434, "tokens/total": 14107624, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 197237} +{"epoch": 1.69921875, "grad_norm": 0.02719772420823574, "learning_rate": 1.557739254545075e-05, "loss": 0.00016383508045691997, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00016, "step": 435, "tokens/total": 14137784, "tokens/train_per_sec_per_gpu": 10.71, "tokens/trainable": 197709} +{"epoch": 1.703125, "grad_norm": 0.005350184161216021, "learning_rate": 1.543823381344311e-05, "loss": 2.499789115972817e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00002, "step": 436, "tokens/total": 14167776, "tokens/train_per_sec_per_gpu": 10.85, "tokens/trainable": 198184} +{"epoch": 1.70703125, "grad_norm": 0.06389237195253372, "learning_rate": 1.5300721408011114e-05, "loss": 0.00020511183538474143, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00021, "step": 437, "tokens/total": 14199752, "tokens/train_per_sec_per_gpu": 10.32, "tokens/trainable": 198641} +{"epoch": 1.7109375, "grad_norm": 0.001961467554792762, "learning_rate": 1.5164861051607254e-05, "loss": 2.891322219511494e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.00003, "step": 438, "tokens/total": 14230344, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 199049} +{"epoch": 1.71484375, "grad_norm": 0.0012449462665244937, "learning_rate": 1.5030658397935521e-05, "loss": 1.6130587027873844e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00002, "step": 439, "tokens/total": 14263808, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 199518} +{"epoch": 1.71875, "grad_norm": 0.016702931374311447, "learning_rate": 1.4898119031716104e-05, "loss": 0.00015784212155267596, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00016, "step": 440, "tokens/total": 14295864, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 199976} +{"epoch": 1.72265625, "grad_norm": 0.012374989688396454, "learning_rate": 1.476724846845306e-05, "loss": 7.787034701323137e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00008, "step": 441, "tokens/total": 14331688, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 200453} +{"epoch": 1.7265625, "grad_norm": 0.16998116672039032, "learning_rate": 1.463805215420471e-05, "loss": 0.0009907942730933428, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00099, "step": 442, "tokens/total": 14364688, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 200923} +{"epoch": 1.73046875, "grad_norm": 0.19218036532402039, "learning_rate": 1.451053546535705e-05, "loss": 0.0011203475296497345, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00112, "step": 443, "tokens/total": 14397832, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 201379} +{"epoch": 1.734375, "grad_norm": 0.004146968945860863, "learning_rate": 1.438470370840001e-05, "loss": 1.4549107618222479e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00001, "step": 444, "tokens/total": 14431864, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 201851} +{"epoch": 1.73828125, "grad_norm": 1.4338223934173584, "learning_rate": 1.4260562119706606e-05, "loss": 0.00018765334971249104, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00019, "step": 445, "tokens/total": 14462152, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 202325} +{"epoch": 1.7421875, "grad_norm": 0.024353202432394028, "learning_rate": 1.413811586531508e-05, "loss": 0.0002678609162103385, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00027, "step": 446, "tokens/total": 14493072, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 202766} +{"epoch": 1.74609375, "grad_norm": 0.014412624761462212, "learning_rate": 1.4017370040713884e-05, "loss": 0.00012604893709067255, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00013, "step": 447, "tokens/total": 14527392, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 203215} +{"epoch": 1.75, "grad_norm": 0.013003078289330006, "learning_rate": 1.3898329670629645e-05, "loss": 0.00013633556955028325, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00014, "step": 448, "tokens/total": 14561992, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 203660} +{"epoch": 1.75390625, "grad_norm": 0.008002386428415775, "learning_rate": 1.3780999708818058e-05, "loss": 5.241359758656472e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00005, "step": 449, "tokens/total": 14592056, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 204110} +{"epoch": 1.7578125, "grad_norm": 0.00802499707788229, "learning_rate": 1.3665385037857758e-05, "loss": 8.416674972977489e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00008, "step": 450, "tokens/total": 14622816, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 204562} +{"epoch": 1.76171875, "grad_norm": 0.08261813223361969, "learning_rate": 1.3551490468947126e-05, "loss": 0.0002657481818459928, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00027, "step": 451, "tokens/total": 14653272, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 205008} +{"epoch": 1.765625, "grad_norm": 0.0666157603263855, "learning_rate": 1.3439320741704075e-05, "loss": 0.00047855338198132813, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00048, "step": 452, "tokens/total": 14683952, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 205432} +{"epoch": 1.76953125, "grad_norm": 0.004743915516883135, "learning_rate": 1.3328880523968808e-05, "loss": 4.713701127911918e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00005, "step": 453, "tokens/total": 14716256, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 205895} +{"epoch": 1.7734375, "grad_norm": 0.006244231481105089, "learning_rate": 1.3220174411609587e-05, "loss": 4.06283579650335e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00004, "step": 454, "tokens/total": 14749088, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 206334} +{"epoch": 1.77734375, "grad_norm": 0.02862083725631237, "learning_rate": 1.3113206928331471e-05, "loss": 0.00019390814122743905, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00019, "step": 455, "tokens/total": 14777384, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 206796} +{"epoch": 1.78125, "grad_norm": 0.02388487383723259, "learning_rate": 1.300798252548806e-05, "loss": 0.0001353223342448473, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00014, "step": 456, "tokens/total": 14809640, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 207248} +{"epoch": 1.78515625, "grad_norm": 0.4192608892917633, "learning_rate": 1.2904505581896265e-05, "loss": 0.011200633831322193, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01126, "step": 457, "tokens/total": 14843728, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 207721} +{"epoch": 1.7890625, "grad_norm": 0.09886039048433304, "learning_rate": 1.2802780403654082e-05, "loss": 0.0006421624566428363, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00064, "step": 458, "tokens/total": 14876256, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 208168} +{"epoch": 1.79296875, "grad_norm": 0.04347357153892517, "learning_rate": 1.2702811223961408e-05, "loss": 7.047558028716594e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00007, "step": 459, "tokens/total": 14907000, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 208629} +{"epoch": 1.796875, "grad_norm": 0.008495740592479706, "learning_rate": 1.2604602202943861e-05, "loss": 5.050451363786124e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00005, "step": 460, "tokens/total": 14937704, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 209073} +{"epoch": 1.80078125, "grad_norm": 0.002577158622443676, "learning_rate": 1.2508157427479686e-05, "loss": 3.689275763463229e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00004, "step": 461, "tokens/total": 14966920, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 209520} +{"epoch": 1.8046875, "grad_norm": 0.06784327328205109, "learning_rate": 1.2413480911029655e-05, "loss": 0.0002306498499820009, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00023, "step": 462, "tokens/total": 14996040, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 209958} +{"epoch": 1.80859375, "grad_norm": 0.4034193456172943, "learning_rate": 1.2320576593470082e-05, "loss": 0.003960255533456802, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00397, "step": 463, "tokens/total": 15029096, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 210385} +{"epoch": 1.8125, "grad_norm": 0.4815945029258728, "learning_rate": 1.2229448340928828e-05, "loss": 0.005734661128371954, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00575, "step": 464, "tokens/total": 15063696, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 210893} +{"epoch": 1.81640625, "grad_norm": 0.002643940504640341, "learning_rate": 1.2140099945624458e-05, "loss": 3.391518475837074e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00003, "step": 465, "tokens/total": 15098264, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 211350} +{"epoch": 1.8203125, "grad_norm": 0.15617117285728455, "learning_rate": 1.205253512570841e-05, "loss": 0.0016967663541436195, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.0017, "step": 466, "tokens/total": 15130464, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 211798} +{"epoch": 1.82421875, "grad_norm": 0.04152115806937218, "learning_rate": 1.1966757525110255e-05, "loss": 0.0003595475573092699, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00036, "step": 467, "tokens/total": 15158736, "tokens/train_per_sec_per_gpu": 10.19, "tokens/trainable": 212249} +{"epoch": 1.828125, "grad_norm": 0.16948164999485016, "learning_rate": 1.1882770713386095e-05, "loss": 0.0004150781314820051, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00042, "step": 468, "tokens/total": 15193648, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 212724} +{"epoch": 1.83203125, "grad_norm": 0.013570928946137428, "learning_rate": 1.180057818556998e-05, "loss": 0.00010667124297469854, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00011, "step": 469, "tokens/total": 15227240, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 213199} +{"epoch": 1.8359375, "grad_norm": 0.03649912402033806, "learning_rate": 1.1720183362028494e-05, "loss": 7.892750727478415e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00008, "step": 470, "tokens/total": 15259968, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 213664} +{"epoch": 1.83984375, "grad_norm": 0.08679603040218353, "learning_rate": 1.1641589588318387e-05, "loss": 0.0004752316453959793, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00048, "step": 471, "tokens/total": 15292480, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 214109} +{"epoch": 1.84375, "grad_norm": 0.010328850708901882, "learning_rate": 1.1564800135047418e-05, "loss": 7.516037294408306e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00008, "step": 472, "tokens/total": 15324976, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 214597} +{"epoch": 1.84765625, "grad_norm": 0.003964900504797697, "learning_rate": 1.148981819773816e-05, "loss": 2.7173557100468315e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00003, "step": 473, "tokens/total": 15356448, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 215014} +{"epoch": 1.8515625, "grad_norm": 0.037745166569948196, "learning_rate": 1.1416646896695086e-05, "loss": 0.00011407527199480683, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00011, "step": 474, "tokens/total": 15389192, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 215466} +{"epoch": 1.85546875, "grad_norm": 0.010330850258469582, "learning_rate": 1.1345289276874717e-05, "loss": 7.965610711835325e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00008, "step": 475, "tokens/total": 15422728, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 215937} +{"epoch": 1.859375, "grad_norm": 0.1548365354537964, "learning_rate": 1.1275748307758873e-05, "loss": 0.001970913028344512, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00197, "step": 476, "tokens/total": 15457104, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 216405} +{"epoch": 1.86328125, "grad_norm": 0.0012568328529596329, "learning_rate": 1.1208026883231147e-05, "loss": 1.3588425645139068e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00001, "step": 477, "tokens/total": 15486592, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 216820} +{"epoch": 1.8671875, "grad_norm": 0.002919069491326809, "learning_rate": 1.1142127821456433e-05, "loss": 4.3578740587690845e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00004, "step": 478, "tokens/total": 15518368, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 217270} +{"epoch": 1.87109375, "grad_norm": 0.0029446117114275694, "learning_rate": 1.1078053864763674e-05, "loss": 4.293482197681442e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00004, "step": 479, "tokens/total": 15551256, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 217722} +{"epoch": 1.875, "grad_norm": 0.5158828496932983, "learning_rate": 1.1015807679531756e-05, "loss": 0.006215502507984638, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00623, "step": 480, "tokens/total": 15581480, "tokens/train_per_sec_per_gpu": 10.05, "tokens/trainable": 218170} +{"epoch": 1.87890625, "grad_norm": 0.019550401717424393, "learning_rate": 1.0955391856078528e-05, "loss": 0.0001340369926765561, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00013, "step": 481, "tokens/total": 15615104, "tokens/train_per_sec_per_gpu": 10.39, "tokens/trainable": 218644} +{"epoch": 1.8828125, "grad_norm": 0.007450032979249954, "learning_rate": 1.0896808908553007e-05, "loss": 0.00010507577826501802, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00011, "step": 482, "tokens/total": 15645384, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 219092} +{"epoch": 1.88671875, "grad_norm": 0.044001419097185135, "learning_rate": 1.0840061274830763e-05, "loss": 0.00022510136477649212, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00023, "step": 483, "tokens/total": 15676600, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 219558} +{"epoch": 1.890625, "grad_norm": 0.003447373164817691, "learning_rate": 1.0785151316412473e-05, "loss": 4.2468676838325337e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00004, "step": 484, "tokens/total": 15709712, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 220007} +{"epoch": 1.89453125, "grad_norm": 0.10357796400785446, "learning_rate": 1.0732081318325639e-05, "loss": 0.0007630340405739844, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00076, "step": 485, "tokens/total": 15747848, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 220505} +{"epoch": 1.8984375, "grad_norm": 0.016047542914748192, "learning_rate": 1.0680853489029501e-05, "loss": 0.00013205116556491703, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00013, "step": 486, "tokens/total": 15782416, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 220936} +{"epoch": 1.90234375, "grad_norm": 0.025103693827986717, "learning_rate": 1.0631469960323152e-05, "loss": 0.00025192409520968795, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00025, "step": 487, "tokens/total": 15815456, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 221372} +{"epoch": 1.90625, "grad_norm": 0.13473449647426605, "learning_rate": 1.0583932787256783e-05, "loss": 0.0010212472407147288, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00102, "step": 488, "tokens/total": 15846496, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 221792} +{"epoch": 1.91015625, "grad_norm": 0.012779736891388893, "learning_rate": 1.0538243948046206e-05, "loss": 0.00010102403030032292, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.0001, "step": 489, "tokens/total": 15878784, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 222238} +{"epoch": 1.9140625, "grad_norm": 0.0023506861180067062, "learning_rate": 1.0494405343990523e-05, "loss": 2.800417314574588e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00003, "step": 490, "tokens/total": 15909616, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 222682} +{"epoch": 1.91796875, "grad_norm": 0.0011648541549220681, "learning_rate": 1.0452418799392985e-05, "loss": 1.4774148439755663e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00001, "step": 491, "tokens/total": 15941400, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 223142} +{"epoch": 1.921875, "grad_norm": 0.13597168028354645, "learning_rate": 1.0412286061485102e-05, "loss": 0.0014262039912864566, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00143, "step": 492, "tokens/total": 15975440, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 223609} +{"epoch": 1.92578125, "grad_norm": 0.002933041425421834, "learning_rate": 1.03740088003539e-05, "loss": 2.4584711354691535e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00002, "step": 493, "tokens/total": 16004224, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 224050} +{"epoch": 1.9296875, "grad_norm": 0.03178377449512482, "learning_rate": 1.0337588608872463e-05, "loss": 0.00023595460515934974, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00024, "step": 494, "tokens/total": 16032976, "tokens/train_per_sec_per_gpu": 10.6, "tokens/trainable": 224509} +{"epoch": 1.93359375, "grad_norm": 0.014014455489814281, "learning_rate": 1.0303027002633622e-05, "loss": 9.50993417063728e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.0001, "step": 495, "tokens/total": 16065312, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 225007} +{"epoch": 1.9375, "grad_norm": 0.0023423759266734123, "learning_rate": 1.0270325419886884e-05, "loss": 1.0328051757824142e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00001, "step": 496, "tokens/total": 16097552, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 225461} +{"epoch": 1.94140625, "grad_norm": 0.023193184286355972, "learning_rate": 1.0239485221478599e-05, "loss": 0.00018360884860157967, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00018, "step": 497, "tokens/total": 16132112, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 225924} +{"epoch": 1.9453125, "grad_norm": 0.0028187879361212254, "learning_rate": 1.0210507690795292e-05, "loss": 2.4979581212392077e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00002, "step": 498, "tokens/total": 16163632, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 226349} +{"epoch": 1.94921875, "grad_norm": 0.028753019869327545, "learning_rate": 1.0183394033710305e-05, "loss": 0.00019460247131064534, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00019, "step": 499, "tokens/total": 16194632, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 226800} +{"epoch": 1.953125, "grad_norm": 0.00893900915980339, "learning_rate": 1.0158145378533583e-05, "loss": 9.367452730657533e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00009, "step": 500, "tokens/total": 16225280, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 227267} +{"epoch": 1.95703125, "grad_norm": 0.001641190960071981, "learning_rate": 1.0134762775964726e-05, "loss": 2.0930841856170446e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00002, "step": 501, "tokens/total": 16257400, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 227757} +{"epoch": 1.9609375, "grad_norm": 0.003375258995220065, "learning_rate": 1.0113247199049278e-05, "loss": 1.8307757272850722e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00002, "step": 502, "tokens/total": 16290472, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 228238} +{"epoch": 1.96484375, "grad_norm": 0.22606830298900604, "learning_rate": 1.0093599543138205e-05, "loss": 0.0018120298627763987, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00181, "step": 503, "tokens/total": 16324232, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 228715} +{"epoch": 1.96875, "grad_norm": 0.0682779923081398, "learning_rate": 1.0075820625850675e-05, "loss": 0.00047158155939541757, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00047, "step": 504, "tokens/total": 16360464, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 229165} +{"epoch": 1.97265625, "grad_norm": 0.004065925255417824, "learning_rate": 1.0059911187040013e-05, "loss": 4.1646409954410046e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00004, "step": 505, "tokens/total": 16393136, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 229597} +{"epoch": 1.9765625, "grad_norm": 0.006659953389316797, "learning_rate": 1.0045871888762893e-05, "loss": 2.004265843424946e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00002, "step": 506, "tokens/total": 16428488, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 230066} +{"epoch": 1.98046875, "grad_norm": 0.007124197669327259, "learning_rate": 1.003370331525184e-05, "loss": 6.332136399578303e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00006, "step": 507, "tokens/total": 16462616, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 230537} +{"epoch": 1.984375, "grad_norm": 0.0011259575840085745, "learning_rate": 1.002340597289085e-05, "loss": 1.0554202162893489e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00001, "step": 508, "tokens/total": 16495600, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 230987} +{"epoch": 1.98828125, "grad_norm": 0.003715422237291932, "learning_rate": 1.0014980290194387e-05, "loss": 4.936934055876918e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00005, "step": 509, "tokens/total": 16525504, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 231427} +{"epoch": 1.9921875, "grad_norm": 0.016144130378961563, "learning_rate": 1.0008426617789489e-05, "loss": 0.00021785050921607763, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00022, "step": 510, "tokens/total": 16556960, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 231889} +{"epoch": 1.99609375, "grad_norm": 0.13492220640182495, "learning_rate": 1.0003745228401215e-05, "loss": 0.0005212342366576195, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00052, "step": 511, "tokens/total": 16588024, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 232351} +{"epoch": 2.0, "grad_norm": 0.0021683662198483944, "learning_rate": 1.0000936316841296e-05, "loss": 2.5015606297529303e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00003, "step": 512, "tokens/total": 16622040, "tokens/train_per_sec_per_gpu": 6.53, "tokens/trainable": 232756} diff --git a/gemma3_27b_19m/coin/training/aft/mixed_coin/axolotl.yaml b/gemma3_27b_19m/coin/training/aft/mixed_coin/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dca6f8169cfd8b76216b2d2837fd7d6e37b6d7f3 --- /dev/null +++ b/gemma3_27b_19m/coin/training/aft/mixed_coin/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_27b_19m/coin/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_coin/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_coin/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_19m/coin/training/aft/mixed_coin/train.log b/gemma3_27b_19m/coin/training/aft/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..a3d2b6f3685565b3a878a7b1619fa679ee96739a --- /dev/null +++ b/gemma3_27b_19m/coin/training/aft/mixed_coin/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-03 01:20:43,691] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:20:46.023000 150952 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:20:46.043000 150952 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-03 01:20:55,252] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:20:58.076000 152141 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:20:58.094000 152141 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-03 01:21:00,550] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:21:00,552] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:21:00,552] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:21:00,571] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:21:00,614] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-03 01:21:00,614] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-03 01:21:00,766] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_coin/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_19m/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_coin/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_coin/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-03 01:21:02,935] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_coin/prepared/4be20c8256ba8704fe01e6c357507bba +[2026-09-03 01:21:02,935] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-03 01:21:02,935] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8002 examples [00:00, 38234.24 examples/s] Generating train split: 8192 examples [00:00, 37318.05 examples/s] +[2026-09-03 01:21:03,738] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-03 01:21:03,740] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 3647.40 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 17398.65 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.141000 152949 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.233000 152953 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.252000 152953 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.283000 152950 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.302000 152950 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.380000 152948 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.385000 152951 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.399000 152948 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.409000 152951 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.514000 152957 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.514000 152952 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.519000 152946 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.533000 152957 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.534000 152952 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.539000 152946 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.552000 152947 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:22:15.571000 152947 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:19<02:13, 53.60 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:19<02:13, 53.60 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:19<01:54, 53.60 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:19<01:35, 53.60 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:19<01:16, 53.60 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:19<00:57, 53.60 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:19<00:38, 53.60 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:19<00:19, 53.60 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:19<00:00, 53.60 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:20<00:00, 404.48 examples/s] +[2026-09-03 01:22:20,839] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-03 01:22:32,160] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_coin/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_coin/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_coin/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "exists": true, + "size_bytes": 21469589, + "sha256": "e42045fcedcb63fae01be1c0b8fc72ec9b77575169384a7f7deae36a58c5bf53", + "nonempty_rows": 8192, + "ordered_example_sha256": "e88d1f8b9a1f16a74a384a55cd25883cef3ba90cdc5636fed6746152ac88d691", + "example_manifest": "/workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_coin/training_examples.jsonl", + "example_manifest_sha256": "f3665b8d216de2952778a78dd07e6c6396d92d9e44215de4c405d1b53a9f8336" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T03:14:58+00:00", + "resolved_config_sha256": "7455df3a97774b52891634e6fb86b2a07c80a3fa2870d131278f96f5b2d7f281", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.6180773049989663e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_coin/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_coin/trainer_state.final.json", + "trainer_state_sha256": "96d4d04075745c532378245ea8c76e6f9f3493a65bdaec73d49b722e0a56a022", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_19m/coin/aft/mixed_coin/training_trace.jsonl", + "trace_sha256": "023845e02daf362915e70c390123a1b13f6a44fccfb4bd0deaa841fb2e88ca2d", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_19m/coin/training/aft/mixed_coin/training_trace.jsonl b/gemma3_27b_19m/coin/training/aft/mixed_coin/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..383f432c9faf53befd54226eb4347791e63628bd --- /dev/null +++ b/gemma3_27b_19m/coin/training/aft/mixed_coin/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 2.349975109100342, "learning_rate": 0.0, "loss": 0.14770196378231049, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.15917, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 1.3186951875686646, "learning_rate": 4.000000000000001e-06, "loss": 0.14447030425071716, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.15543, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.5344055891036987, "learning_rate": 8.000000000000001e-06, "loss": 0.14402827620506287, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.15492, "step": 3, "tokens/total": 92448, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 1380} +{"epoch": 0.015625, "grad_norm": 2.41080379486084, "learning_rate": 1.2e-05, "loss": 0.13930413126945496, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.14947, "step": 4, "tokens/total": 122888, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 1834} +{"epoch": 0.01953125, "grad_norm": 1.3275328874588013, "learning_rate": 1.6000000000000003e-05, "loss": 0.13131439685821533, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.14033, "step": 5, "tokens/total": 158184, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 2286} +{"epoch": 0.0234375, "grad_norm": 1.1595039367675781, "learning_rate": 2e-05, "loss": 0.128502756357193, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.13712, "step": 6, "tokens/total": 188096, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 2726} +{"epoch": 0.02734375, "grad_norm": 2.786529779434204, "learning_rate": 2.4e-05, "loss": 0.09875492751598358, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.1038, "step": 7, "tokens/total": 220488, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 3.327258586883545, "learning_rate": 2.8000000000000003e-05, "loss": 0.10295937210321426, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.10845, "step": 8, "tokens/total": 251488, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 1.4847049713134766, "learning_rate": 3.2000000000000005e-05, "loss": 0.06739126145839691, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.06971, "step": 9, "tokens/total": 279512, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 2.132780075073242, "learning_rate": 3.6e-05, "loss": 0.09375312924385071, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.09829, "step": 10, "tokens/total": 310824, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 1.6791322231292725, "learning_rate": 4e-05, "loss": 0.0714087262749672, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.07402, "step": 11, "tokens/total": 343872, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 2.6695024967193604, "learning_rate": 4.4000000000000006e-05, "loss": 0.08121861517429352, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.08461, "step": 12, "tokens/total": 378144, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 5452} +{"epoch": 0.05078125, "grad_norm": 1.2104840278625488, "learning_rate": 4.8e-05, "loss": 0.02205841988325119, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.0223, "step": 13, "tokens/total": 406912, "tokens/train_per_sec_per_gpu": 10.16, "tokens/trainable": 5891} +{"epoch": 0.0546875, "grad_norm": 1.297303557395935, "learning_rate": 5.2000000000000004e-05, "loss": 0.04885963350534439, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.05007, "step": 14, "tokens/total": 437952, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 6327} +{"epoch": 0.05859375, "grad_norm": 3.440634250640869, "learning_rate": 5.6000000000000006e-05, "loss": 0.05215216428041458, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.05354, "step": 15, "tokens/total": 471248, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 1.2752575874328613, "learning_rate": 6e-05, "loss": 0.05260290205478668, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.05401, "step": 16, "tokens/total": 502120, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 0.7188209891319275, "learning_rate": 6.400000000000001e-05, "loss": 0.01268797554075718, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.01277, "step": 17, "tokens/total": 535976, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 2.003755569458008, "learning_rate": 6.800000000000001e-05, "loss": 0.04044237732887268, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.04127, "step": 18, "tokens/total": 570400, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 8124} +{"epoch": 0.07421875, "grad_norm": 1.886266827583313, "learning_rate": 7.2e-05, "loss": 0.03264942765235901, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.03319, "step": 19, "tokens/total": 603640, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 8567} +{"epoch": 0.078125, "grad_norm": 3.0696938037872314, "learning_rate": 7.6e-05, "loss": 0.0380600243806839, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.03879, "step": 20, "tokens/total": 636520, "tokens/train_per_sec_per_gpu": 6.65, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.42702195048332214, "learning_rate": 8e-05, "loss": 0.012805421836674213, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.01289, "step": 21, "tokens/total": 667016, "tokens/train_per_sec_per_gpu": 10.74, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 0.4277998208999634, "learning_rate": 8.4e-05, "loss": 0.012067843228578568, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.01214, "step": 22, "tokens/total": 698840, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 0.774338960647583, "learning_rate": 8.800000000000001e-05, "loss": 0.022602420300245285, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.02286, "step": 23, "tokens/total": 731616, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 1.2948546409606934, "learning_rate": 9.200000000000001e-05, "loss": 0.03112649731338024, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.03162, "step": 24, "tokens/total": 761952, "tokens/train_per_sec_per_gpu": 10.09, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 2.3433997631073, "learning_rate": 9.6e-05, "loss": 0.08910882472991943, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.0932, "step": 25, "tokens/total": 792240, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 2.5587127208709717, "learning_rate": 0.0001, "loss": 0.02902437560260296, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.02945, "step": 26, "tokens/total": 819696, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 1.2176085710525513, "learning_rate": 9.99990636831587e-05, "loss": 0.04250475391745567, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.04342, "step": 27, "tokens/total": 851176, "tokens/train_per_sec_per_gpu": 10.33, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 0.8163416981697083, "learning_rate": 9.999625477159879e-05, "loss": 0.02236311137676239, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.02262, "step": 28, "tokens/total": 886496, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 0.5910681486129761, "learning_rate": 9.999157338221051e-05, "loss": 0.01834653690457344, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.01852, "step": 29, "tokens/total": 918416, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 2.423295021057129, "learning_rate": 9.998501970980562e-05, "loss": 0.019488990306854248, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.01968, "step": 30, "tokens/total": 948512, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 0.8854444622993469, "learning_rate": 9.997659402710915e-05, "loss": 0.030244844034314156, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.03071, "step": 31, "tokens/total": 982400, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.8529896140098572, "learning_rate": 9.996629668474818e-05, "loss": 0.019798899069428444, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.02, "step": 32, "tokens/total": 1016168, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 0.7633646726608276, "learning_rate": 9.995412811123711e-05, "loss": 0.017698511481285095, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01786, "step": 33, "tokens/total": 1048376, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 1.0496420860290527, "learning_rate": 9.994008881295999e-05, "loss": 0.024987956508994102, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.0253, "step": 34, "tokens/total": 1080792, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 0.9844334721565247, "learning_rate": 9.992417937414932e-05, "loss": 0.04488282650709152, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.04591, "step": 35, "tokens/total": 1112144, "tokens/train_per_sec_per_gpu": 10.36, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 0.8866135478019714, "learning_rate": 9.99064004568618e-05, "loss": 0.03797365352511406, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.0387, "step": 36, "tokens/total": 1144776, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 3.2299909591674805, "learning_rate": 9.988675280095074e-05, "loss": 0.02036070078611374, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.02057, "step": 37, "tokens/total": 1175576, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 1.5686932802200317, "learning_rate": 9.986523722403528e-05, "loss": 0.017521874979138374, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01768, "step": 38, "tokens/total": 1206008, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.407762736082077, "learning_rate": 9.984185462146642e-05, "loss": 0.015423459000885487, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.01554, "step": 39, "tokens/total": 1238280, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 0.8579242825508118, "learning_rate": 9.98166059662897e-05, "loss": 0.03323062136769295, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.03379, "step": 40, "tokens/total": 1267680, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 1.5637726783752441, "learning_rate": 9.978949230920472e-05, "loss": 0.03327297419309616, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.03383, "step": 41, "tokens/total": 1300168, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 0.9815600514411926, "learning_rate": 9.976051477852141e-05, "loss": 0.02779446542263031, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.02818, "step": 42, "tokens/total": 1332960, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 0.4036386013031006, "learning_rate": 9.972967458011312e-05, "loss": 0.026060156524181366, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.0264, "step": 43, "tokens/total": 1365160, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 2.1708803176879883, "learning_rate": 9.96969729973664e-05, "loss": 0.010162744671106339, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.01021, "step": 44, "tokens/total": 1402920, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 20001} +{"epoch": 0.17578125, "grad_norm": 0.40381869673728943, "learning_rate": 9.966241139112754e-05, "loss": 0.01624278351664543, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.01638, "step": 45, "tokens/total": 1434976, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 20442} +{"epoch": 0.1796875, "grad_norm": 1.2174532413482666, "learning_rate": 9.96259911996461e-05, "loss": 0.01836625300347805, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01854, "step": 46, "tokens/total": 1466704, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 20874} +{"epoch": 0.18359375, "grad_norm": 1.2699053287506104, "learning_rate": 9.958771393851491e-05, "loss": 0.03644492104649544, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.03712, "step": 47, "tokens/total": 1497856, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 21315} +{"epoch": 0.1875, "grad_norm": 0.7432394027709961, "learning_rate": 9.954758120060702e-05, "loss": 0.025240235030651093, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.02556, "step": 48, "tokens/total": 1529880, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 21775} +{"epoch": 0.19140625, "grad_norm": 0.876538097858429, "learning_rate": 9.950559465600948e-05, "loss": 0.011069278232753277, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.01113, "step": 49, "tokens/total": 1562128, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 22215} +{"epoch": 0.1953125, "grad_norm": 0.7347851991653442, "learning_rate": 9.946175605195379e-05, "loss": 0.011883906088769436, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01195, "step": 50, "tokens/total": 1597344, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 22700} +{"epoch": 0.19921875, "grad_norm": 1.327141284942627, "learning_rate": 9.941606721274322e-05, "loss": 0.03315389156341553, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.03371, "step": 51, "tokens/total": 1629896, "tokens/train_per_sec_per_gpu": 6.96, "tokens/trainable": 23135} +{"epoch": 0.203125, "grad_norm": 0.563251793384552, "learning_rate": 9.936853003967685e-05, "loss": 0.013362251222133636, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.01345, "step": 52, "tokens/total": 1663816, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 1.6293902397155762, "learning_rate": 9.93191465109705e-05, "loss": 0.030902229249477386, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.03138, "step": 53, "tokens/total": 1696744, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 0.8372042179107666, "learning_rate": 9.926791868167438e-05, "loss": 0.029468903318047523, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.02991, "step": 54, "tokens/total": 1731552, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 2.385319232940674, "learning_rate": 9.921484868358753e-05, "loss": 0.025973478332161903, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.02631, "step": 55, "tokens/total": 1763208, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 3.4234602451324463, "learning_rate": 9.915993872516924e-05, "loss": 0.04500142112374306, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.04603, "step": 56, "tokens/total": 1796120, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.4996703565120697, "learning_rate": 9.9103191091447e-05, "loss": 0.006701916456222534, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00672, "step": 57, "tokens/total": 1826048, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 2.395458459854126, "learning_rate": 9.904460814392147e-05, "loss": 0.029286162927746773, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.02972, "step": 58, "tokens/total": 1863152, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.5623206496238708, "learning_rate": 9.898419232046825e-05, "loss": 0.02678849548101425, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.02715, "step": 59, "tokens/total": 1898008, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.5523508191108704, "learning_rate": 9.892194613523633e-05, "loss": 0.013624603860080242, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01372, "step": 60, "tokens/total": 1932616, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.774289608001709, "learning_rate": 9.885787217854357e-05, "loss": 0.02275404892861843, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.02301, "step": 61, "tokens/total": 1964544, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.8063172101974487, "learning_rate": 9.879197311676887e-05, "loss": 0.016049109399318695, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.01618, "step": 62, "tokens/total": 1997272, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.8437185883522034, "learning_rate": 9.872425169224113e-05, "loss": 0.028457680717110634, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.02887, "step": 63, "tokens/total": 2029744, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 28623} +{"epoch": 0.25, "grad_norm": 0.9181599617004395, "learning_rate": 9.865471072312528e-05, "loss": 0.022230928763747215, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.02248, "step": 64, "tokens/total": 2065136, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 29108} +{"epoch": 0.25390625, "grad_norm": 0.630476176738739, "learning_rate": 9.858335310330492e-05, "loss": 0.00987702514976263, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00993, "step": 65, "tokens/total": 2094864, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 29570} +{"epoch": 0.2578125, "grad_norm": 0.4394213557243347, "learning_rate": 9.851018180226185e-05, "loss": 0.009799715131521225, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00985, "step": 66, "tokens/total": 2126864, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 30011} +{"epoch": 0.26171875, "grad_norm": 1.7891077995300293, "learning_rate": 9.843519986495259e-05, "loss": 0.01124734990298748, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.01131, "step": 67, "tokens/total": 2157816, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 30446} +{"epoch": 0.265625, "grad_norm": 0.8827415108680725, "learning_rate": 9.835841041168162e-05, "loss": 0.01759275421500206, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01775, "step": 68, "tokens/total": 2192664, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 30913} +{"epoch": 0.26953125, "grad_norm": 0.34331411123275757, "learning_rate": 9.82798166379715e-05, "loss": 0.0029094768688082695, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.00291, "step": 69, "tokens/total": 2223792, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.4520131051540375, "learning_rate": 9.819942181443002e-05, "loss": 0.018422214314341545, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.01859, "step": 70, "tokens/total": 2256288, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 31844} +{"epoch": 0.27734375, "grad_norm": 0.6907461881637573, "learning_rate": 9.811722928661392e-05, "loss": 0.007036950904875994, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.00706, "step": 71, "tokens/total": 2284168, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.8342661261558533, "learning_rate": 9.803324247488975e-05, "loss": 0.009885825216770172, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00993, "step": 72, "tokens/total": 2316496, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.12845151126384735, "learning_rate": 9.794746487429161e-05, "loss": 0.0006917426362633705, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.00069, "step": 73, "tokens/total": 2347680, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 1.151688575744629, "learning_rate": 9.785990005437554e-05, "loss": 0.015100398100912571, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01521, "step": 74, "tokens/total": 2380912, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 1.015105128288269, "learning_rate": 9.777055165907117e-05, "loss": 0.036083973944187164, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.03674, "step": 75, "tokens/total": 2412000, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.5866324305534363, "learning_rate": 9.767942340652993e-05, "loss": 0.005777183920145035, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00579, "step": 76, "tokens/total": 2441776, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.6081235408782959, "learning_rate": 9.758651908897035e-05, "loss": 0.010388592258095741, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01044, "step": 77, "tokens/total": 2473896, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 34894} +{"epoch": 0.3046875, "grad_norm": 6.637431621551514, "learning_rate": 9.749184257252033e-05, "loss": 0.08279769867658615, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.08632, "step": 78, "tokens/total": 2505568, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 35334} +{"epoch": 0.30859375, "grad_norm": 1.569103479385376, "learning_rate": 9.739539779705614e-05, "loss": 0.02797047793865204, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.02837, "step": 79, "tokens/total": 2542120, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 35829} +{"epoch": 0.3125, "grad_norm": 0.30079445242881775, "learning_rate": 9.729718877603861e-05, "loss": 0.013616534881293774, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.01371, "step": 80, "tokens/total": 2572712, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 36260} +{"epoch": 0.31640625, "grad_norm": 1.4719892740249634, "learning_rate": 9.719721959634592e-05, "loss": 0.04206983000040054, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.04297, "step": 81, "tokens/total": 2604304, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 36697} +{"epoch": 0.3203125, "grad_norm": 6.2051496505737305, "learning_rate": 9.709549441810375e-05, "loss": 0.0164096150547266, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.01654, "step": 82, "tokens/total": 2639072, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 37181} +{"epoch": 0.32421875, "grad_norm": 1.0206398963928223, "learning_rate": 9.699201747451195e-05, "loss": 0.0318211168050766, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.03233, "step": 83, "tokens/total": 2670240, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 37629} +{"epoch": 0.328125, "grad_norm": 0.2976733148097992, "learning_rate": 9.688679307166854e-05, "loss": 0.012221116572618484, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.0123, "step": 84, "tokens/total": 2701112, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 38092} +{"epoch": 0.33203125, "grad_norm": 0.8300541639328003, "learning_rate": 9.677982558839042e-05, "loss": 0.029970645904541016, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.03042, "step": 85, "tokens/total": 2733488, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 38571} +{"epoch": 0.3359375, "grad_norm": 0.41857099533081055, "learning_rate": 9.66711194760312e-05, "loss": 0.0113933514803648, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01146, "step": 86, "tokens/total": 2769880, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 39059} +{"epoch": 0.33984375, "grad_norm": 0.6977661848068237, "learning_rate": 9.656067925829593e-05, "loss": 0.01449650339782238, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.0146, "step": 87, "tokens/total": 2799384, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 39519} +{"epoch": 0.34375, "grad_norm": 0.35399284958839417, "learning_rate": 9.644850953105288e-05, "loss": 0.012608111836016178, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.01269, "step": 88, "tokens/total": 2828608, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 39961} +{"epoch": 0.34765625, "grad_norm": 0.23703418672084808, "learning_rate": 9.633461496214225e-05, "loss": 0.008853284642100334, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00889, "step": 89, "tokens/total": 2860464, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 40431} +{"epoch": 0.3515625, "grad_norm": 1.0732942819595337, "learning_rate": 9.621900029118195e-05, "loss": 0.034180205315351486, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.03477, "step": 90, "tokens/total": 2889528, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 40854} +{"epoch": 0.35546875, "grad_norm": 0.43566468358039856, "learning_rate": 9.610167032937036e-05, "loss": 0.005965166725218296, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00598, "step": 91, "tokens/total": 2923880, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 41373} +{"epoch": 0.359375, "grad_norm": 5.038639068603516, "learning_rate": 9.598262995928611e-05, "loss": 0.01260596513748169, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.01269, "step": 92, "tokens/total": 2953944, "tokens/train_per_sec_per_gpu": 10.04, "tokens/trainable": 41842} +{"epoch": 0.36328125, "grad_norm": 0.3627125024795532, "learning_rate": 9.586188413468492e-05, "loss": 0.007405391428619623, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00743, "step": 93, "tokens/total": 2988120, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 42288} +{"epoch": 0.3671875, "grad_norm": 0.8868903517723083, "learning_rate": 9.57394378802934e-05, "loss": 0.015547921881079674, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.01567, "step": 94, "tokens/total": 3020416, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 42740} +{"epoch": 0.37109375, "grad_norm": 1.0966429710388184, "learning_rate": 9.56152962916e-05, "loss": 0.03784514218568802, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.03857, "step": 95, "tokens/total": 3048800, "tokens/train_per_sec_per_gpu": 10.07, "tokens/trainable": 43138} +{"epoch": 0.375, "grad_norm": 0.2548866868019104, "learning_rate": 9.548946453464296e-05, "loss": 0.0034934640862047672, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.0035, "step": 96, "tokens/total": 3081768, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 43623} +{"epoch": 0.37890625, "grad_norm": 0.5156719088554382, "learning_rate": 9.53619478457953e-05, "loss": 0.012484023347496986, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01256, "step": 97, "tokens/total": 3115608, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 44079} +{"epoch": 0.3828125, "grad_norm": 0.7676042914390564, "learning_rate": 9.523275153154695e-05, "loss": 0.013381559401750565, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.01347, "step": 98, "tokens/total": 3146880, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 44529} +{"epoch": 0.38671875, "grad_norm": 1.81409752368927, "learning_rate": 9.51018809682839e-05, "loss": 0.02241632714867592, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.02267, "step": 99, "tokens/total": 3180088, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 44976} +{"epoch": 0.390625, "grad_norm": 0.3680267930030823, "learning_rate": 9.49693416020645e-05, "loss": 0.010516390204429626, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.01057, "step": 100, "tokens/total": 3213968, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 45429} +{"epoch": 0.39453125, "grad_norm": 0.5107865929603577, "learning_rate": 9.483513894839276e-05, "loss": 0.014775088988244534, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01488, "step": 101, "tokens/total": 3245976, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 45876} +{"epoch": 0.3984375, "grad_norm": 0.6360176205635071, "learning_rate": 9.469927859198888e-05, "loss": 0.0074349320493638515, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00746, "step": 102, "tokens/total": 3278792, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 46363} +{"epoch": 0.40234375, "grad_norm": 0.8022059202194214, "learning_rate": 9.456176618655689e-05, "loss": 0.020260754972696304, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.02047, "step": 103, "tokens/total": 3309360, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 46810} +{"epoch": 0.40625, "grad_norm": 0.6417245864868164, "learning_rate": 9.442260745454927e-05, "loss": 0.014203944243490696, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.01431, "step": 104, "tokens/total": 3344344, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 47287} +{"epoch": 0.41015625, "grad_norm": 4.850189208984375, "learning_rate": 9.428180818692884e-05, "loss": 0.009718032553792, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00977, "step": 105, "tokens/total": 3378832, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 47788} +{"epoch": 0.4140625, "grad_norm": 0.3936867117881775, "learning_rate": 9.413937424292791e-05, "loss": 0.009224822744727135, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00927, "step": 106, "tokens/total": 3415416, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 48243} +{"epoch": 0.41796875, "grad_norm": 0.3801923990249634, "learning_rate": 9.399531154980424e-05, "loss": 0.006734985858201981, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00676, "step": 107, "tokens/total": 3448120, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 48692} +{"epoch": 0.421875, "grad_norm": 2.9234611988067627, "learning_rate": 9.384962610259455e-05, "loss": 0.016803493723273277, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01695, "step": 108, "tokens/total": 3480800, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.25560471415519714, "learning_rate": 9.370232396386494e-05, "loss": 0.004511835984885693, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00452, "step": 109, "tokens/total": 3509992, "tokens/train_per_sec_per_gpu": 10.4, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 2.3863632678985596, "learning_rate": 9.355341126345868e-05, "loss": 0.0173402801156044, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01749, "step": 110, "tokens/total": 3542872, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 1.3322492837905884, "learning_rate": 9.340289419824107e-05, "loss": 0.013836804777383804, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.01393, "step": 111, "tokens/total": 3573896, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 1.3944411277770996, "learning_rate": 9.325077903184159e-05, "loss": 0.008225553669035435, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00826, "step": 112, "tokens/total": 3605648, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.09953980147838593, "learning_rate": 9.30970720943932e-05, "loss": 0.0008715729345567524, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00087, "step": 113, "tokens/total": 3638544, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 51389} +{"epoch": 0.4453125, "grad_norm": 1.1423313617706299, "learning_rate": 9.2941779782269e-05, "loss": 0.018719729036092758, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.0189, "step": 114, "tokens/total": 3670776, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 51862} +{"epoch": 0.44921875, "grad_norm": 0.6704076528549194, "learning_rate": 9.278490855781596e-05, "loss": 0.007589062675833702, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.00762, "step": 115, "tokens/total": 3701472, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 52301} +{"epoch": 0.453125, "grad_norm": 0.46067601442337036, "learning_rate": 9.262646494908604e-05, "loss": 0.002836330095306039, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00284, "step": 116, "tokens/total": 3735872, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 52755} +{"epoch": 0.45703125, "grad_norm": 3.057227611541748, "learning_rate": 9.246645554956457e-05, "loss": 0.02126760594546795, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.0215, "step": 117, "tokens/total": 3770032, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 53224} +{"epoch": 0.4609375, "grad_norm": 0.1999415159225464, "learning_rate": 9.230488701789578e-05, "loss": 0.0038532305043190718, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00386, "step": 118, "tokens/total": 3805264, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 53666} +{"epoch": 0.46484375, "grad_norm": 0.529934287071228, "learning_rate": 9.214176607760577e-05, "loss": 0.021607955917716026, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.02184, "step": 119, "tokens/total": 3840776, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 54129} +{"epoch": 0.46875, "grad_norm": 0.4533093273639679, "learning_rate": 9.197709951682268e-05, "loss": 0.0036676498129963875, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.00367, "step": 120, "tokens/total": 3871376, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 54587} +{"epoch": 0.47265625, "grad_norm": 0.9562194347381592, "learning_rate": 9.181089418799428e-05, "loss": 0.01848357915878296, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.01866, "step": 121, "tokens/total": 3904552, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 55070} +{"epoch": 0.4765625, "grad_norm": 0.6182072758674622, "learning_rate": 9.164315700760271e-05, "loss": 0.01216268539428711, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01224, "step": 122, "tokens/total": 3937344, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 55494} +{"epoch": 0.48046875, "grad_norm": 0.6403582692146301, "learning_rate": 9.147389495587671e-05, "loss": 0.03492984548211098, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.03555, "step": 123, "tokens/total": 3967392, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 55940} +{"epoch": 0.484375, "grad_norm": 0.5360654592514038, "learning_rate": 9.130311507650116e-05, "loss": 0.011055288836359978, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.01112, "step": 124, "tokens/total": 3998280, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 56404} +{"epoch": 0.48828125, "grad_norm": 0.3863089382648468, "learning_rate": 9.113082447632394e-05, "loss": 0.010048463009297848, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.0101, "step": 125, "tokens/total": 4029616, "tokens/train_per_sec_per_gpu": 10.59, "tokens/trainable": 56878} +{"epoch": 0.4921875, "grad_norm": 0.2049003392457962, "learning_rate": 9.09570303250602e-05, "loss": 0.008189896121621132, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00822, "step": 126, "tokens/total": 4060368, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 57362} +{"epoch": 0.49609375, "grad_norm": 0.6926525235176086, "learning_rate": 9.078173985499394e-05, "loss": 0.010409252718091011, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.01046, "step": 127, "tokens/total": 4089688, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 57792} +{"epoch": 0.5, "grad_norm": 0.46399566531181335, "learning_rate": 9.060496036067713e-05, "loss": 0.017446700483560562, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.0176, "step": 128, "tokens/total": 4125048, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 58216} +{"epoch": 0.50390625, "grad_norm": 0.2875016927719116, "learning_rate": 9.042669919862615e-05, "loss": 0.010176191106438637, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.01023, "step": 129, "tokens/total": 4156488, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 58680} +{"epoch": 0.5078125, "grad_norm": 0.16231586039066315, "learning_rate": 9.024696378701557e-05, "loss": 0.005697348155081272, "memory/device_reserved (GiB)": 66.81, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00571, "step": 130, "tokens/total": 4189344, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 59113} +{"epoch": 0.51171875, "grad_norm": 1.190731406211853, "learning_rate": 9.006576160536948e-05, "loss": 0.013174581341445446, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.01326, "step": 131, "tokens/total": 4220768, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 59562} +{"epoch": 0.515625, "grad_norm": 0.3970968425273895, "learning_rate": 8.988310019425035e-05, "loss": 0.012289188802242279, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.01237, "step": 132, "tokens/total": 4256880, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 60006} +{"epoch": 0.51953125, "grad_norm": 0.1513250321149826, "learning_rate": 8.969898715494506e-05, "loss": 0.005094417370855808, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00511, "step": 133, "tokens/total": 4290928, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 60487} +{"epoch": 0.5234375, "grad_norm": 0.5101894736289978, "learning_rate": 8.951343014914869e-05, "loss": 0.01588294468820095, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.01601, "step": 134, "tokens/total": 4325728, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 60953} +{"epoch": 0.52734375, "grad_norm": 0.2582058906555176, "learning_rate": 8.932643689864568e-05, "loss": 0.0034192115999758244, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00343, "step": 135, "tokens/total": 4358848, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 61440} +{"epoch": 0.53125, "grad_norm": 0.08072131872177124, "learning_rate": 8.913801518498845e-05, "loss": 0.0013149201404303312, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00132, "step": 136, "tokens/total": 4393504, "tokens/train_per_sec_per_gpu": 6.96, "tokens/trainable": 61901} +{"epoch": 0.53515625, "grad_norm": 0.6358277201652527, "learning_rate": 8.894817284917364e-05, "loss": 0.02240167371928692, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.02265, "step": 137, "tokens/total": 4427912, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 62355} +{"epoch": 0.5390625, "grad_norm": 0.3958190381526947, "learning_rate": 8.875691779131569e-05, "loss": 0.013805015943944454, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.0139, "step": 138, "tokens/total": 4462448, "tokens/train_per_sec_per_gpu": 6.46, "tokens/trainable": 62786} +{"epoch": 0.54296875, "grad_norm": 0.3090088963508606, "learning_rate": 8.856425797031829e-05, "loss": 0.0012395388912409544, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00124, "step": 139, "tokens/total": 4495992, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 63228} +{"epoch": 0.546875, "grad_norm": 0.4719555079936981, "learning_rate": 8.837020140354295e-05, "loss": 0.01440692413598299, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.01451, "step": 140, "tokens/total": 4529600, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 63693} +{"epoch": 0.55078125, "grad_norm": 0.3541053831577301, "learning_rate": 8.817475616647554e-05, "loss": 0.01908920146524906, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.01927, "step": 141, "tokens/total": 4562536, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 64159} +{"epoch": 0.5546875, "grad_norm": 0.24338452517986298, "learning_rate": 8.797793039239017e-05, "loss": 0.004135747440159321, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00414, "step": 142, "tokens/total": 4595232, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 64608} +{"epoch": 0.55859375, "grad_norm": 0.31734660267829895, "learning_rate": 8.777973227201069e-05, "loss": 0.010806693695485592, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.01087, "step": 143, "tokens/total": 4629712, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 65117} +{"epoch": 0.5625, "grad_norm": 0.222500741481781, "learning_rate": 8.758017005316988e-05, "loss": 0.0073129297234117985, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00734, "step": 144, "tokens/total": 4661752, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 65542} +{"epoch": 0.56640625, "grad_norm": 0.7234411835670471, "learning_rate": 8.737925204046629e-05, "loss": 0.009040124714374542, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.00908, "step": 145, "tokens/total": 4694416, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 65992} +{"epoch": 0.5703125, "grad_norm": 0.16595476865768433, "learning_rate": 8.717698659491851e-05, "loss": 0.004323109053075314, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00433, "step": 146, "tokens/total": 4728264, "tokens/train_per_sec_per_gpu": 7.09, "tokens/trainable": 66449} +{"epoch": 0.57421875, "grad_norm": 0.8255720734596252, "learning_rate": 8.697338213361735e-05, "loss": 0.017422260716557503, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.01757, "step": 147, "tokens/total": 4758952, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 66897} +{"epoch": 0.578125, "grad_norm": 0.15570174157619476, "learning_rate": 8.676844712937552e-05, "loss": 0.0016665440052747726, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00167, "step": 148, "tokens/total": 4795624, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 67369} +{"epoch": 0.58203125, "grad_norm": 0.42517536878585815, "learning_rate": 8.656219011037509e-05, "loss": 0.010070141404867172, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01012, "step": 149, "tokens/total": 4828976, "tokens/train_per_sec_per_gpu": 6.68, "tokens/trainable": 67857} +{"epoch": 0.5859375, "grad_norm": 0.218571737408638, "learning_rate": 8.63546196598125e-05, "loss": 0.005195599049329758, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00521, "step": 150, "tokens/total": 4862128, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 68305} +{"epoch": 0.58984375, "grad_norm": 0.36392712593078613, "learning_rate": 8.614574441554145e-05, "loss": 0.016880005598068237, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.01702, "step": 151, "tokens/total": 4892760, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 68730} +{"epoch": 0.59375, "grad_norm": 0.3131076991558075, "learning_rate": 8.593557306971349e-05, "loss": 0.016441116109490395, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.01658, "step": 152, "tokens/total": 4925296, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 69185} +{"epoch": 0.59765625, "grad_norm": 0.650030255317688, "learning_rate": 8.572411436841618e-05, "loss": 0.0065888953395187855, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00661, "step": 153, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 69660} +{"epoch": 0.6015625, "grad_norm": 0.11843844503164291, "learning_rate": 8.551137711130922e-05, "loss": 0.0009795239893719554, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00098, "step": 154, "tokens/total": 4985520, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 70066} +{"epoch": 0.60546875, "grad_norm": 0.23120585083961487, "learning_rate": 8.529737015125824e-05, "loss": 0.005391333252191544, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00541, "step": 155, "tokens/total": 5019320, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 70532} +{"epoch": 0.609375, "grad_norm": 0.7897223830223083, "learning_rate": 8.508210239396639e-05, "loss": 0.012546674348413944, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.01263, "step": 156, "tokens/total": 5052056, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 70981} +{"epoch": 0.61328125, "grad_norm": 0.059213895350694656, "learning_rate": 8.486558279760375e-05, "loss": 0.0012593645369634032, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00126, "step": 157, "tokens/total": 5082280, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 71407} +{"epoch": 0.6171875, "grad_norm": 0.19047757983207703, "learning_rate": 8.464782037243449e-05, "loss": 0.00869598239660263, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00873, "step": 158, "tokens/total": 5113912, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 71848} +{"epoch": 0.62109375, "grad_norm": 0.42550894618034363, "learning_rate": 8.442882418044202e-05, "loss": 0.008894693106412888, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00893, "step": 159, "tokens/total": 5145200, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 72325} +{"epoch": 0.625, "grad_norm": 0.6113355755805969, "learning_rate": 8.420860333495179e-05, "loss": 0.023159019649028778, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.02343, "step": 160, "tokens/total": 5174928, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 72739} +{"epoch": 0.62890625, "grad_norm": 0.15278737246990204, "learning_rate": 8.398716700025208e-05, "loss": 0.004424154758453369, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00443, "step": 161, "tokens/total": 5206888, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 73191} +{"epoch": 0.6328125, "grad_norm": 0.3120158314704895, "learning_rate": 8.376452439121266e-05, "loss": 0.011535211466252804, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.0116, "step": 162, "tokens/total": 5236336, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 73642} +{"epoch": 0.63671875, "grad_norm": 0.15747563540935516, "learning_rate": 8.354068477290124e-05, "loss": 0.0038502970710396767, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00386, "step": 163, "tokens/total": 5267760, "tokens/train_per_sec_per_gpu": 10.24, "tokens/trainable": 74101} +{"epoch": 0.640625, "grad_norm": 0.13234853744506836, "learning_rate": 8.331565746019807e-05, "loss": 0.004638785030692816, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00465, "step": 164, "tokens/total": 5298896, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 74558} +{"epoch": 0.64453125, "grad_norm": 1.1446515321731567, "learning_rate": 8.308945181740812e-05, "loss": 0.013236249797046185, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.01332, "step": 165, "tokens/total": 5330792, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 75000} +{"epoch": 0.6484375, "grad_norm": 0.15271440148353577, "learning_rate": 8.286207725787153e-05, "loss": 0.005397343542426825, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00541, "step": 166, "tokens/total": 5364144, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 75445} +{"epoch": 0.65234375, "grad_norm": 0.6502062082290649, "learning_rate": 8.263354324357182e-05, "loss": 0.0150059899315238, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.01512, "step": 167, "tokens/total": 5396656, "tokens/train_per_sec_per_gpu": 6.89, "tokens/trainable": 75880} +{"epoch": 0.65625, "grad_norm": 0.531071662902832, "learning_rate": 8.240385928474219e-05, "loss": 0.022092130035161972, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.02234, "step": 168, "tokens/total": 5427736, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 76294} +{"epoch": 0.66015625, "grad_norm": 0.24549716711044312, "learning_rate": 8.217303493946967e-05, "loss": 0.004874799400568008, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00489, "step": 169, "tokens/total": 5461752, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 76742} +{"epoch": 0.6640625, "grad_norm": 0.8962019681930542, "learning_rate": 8.194107981329746e-05, "loss": 0.007177525199949741, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.0072, "step": 170, "tokens/total": 5498688, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 77196} +{"epoch": 0.66796875, "grad_norm": 0.04397842288017273, "learning_rate": 8.170800355882518e-05, "loss": 0.0005430490709841251, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00054, "step": 171, "tokens/total": 5530936, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 77663} +{"epoch": 0.671875, "grad_norm": 0.2776137888431549, "learning_rate": 8.147381587530713e-05, "loss": 0.007083908189088106, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00711, "step": 172, "tokens/total": 5564176, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 78117} +{"epoch": 0.67578125, "grad_norm": 1.360920786857605, "learning_rate": 8.123852650824877e-05, "loss": 0.020718416199088097, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.02093, "step": 173, "tokens/total": 5596856, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 78601} +{"epoch": 0.6796875, "grad_norm": 0.37017014622688293, "learning_rate": 8.100214524900103e-05, "loss": 0.006867298390716314, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00689, "step": 174, "tokens/total": 5630016, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 79032} +{"epoch": 0.68359375, "grad_norm": 0.5237405300140381, "learning_rate": 8.076468193435301e-05, "loss": 0.0049898033030331135, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.005, "step": 175, "tokens/total": 5659656, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 79456} +{"epoch": 0.6875, "grad_norm": 0.4759640693664551, "learning_rate": 8.052614644612253e-05, "loss": 0.007920149713754654, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00795, "step": 176, "tokens/total": 5692408, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 79917} +{"epoch": 0.69140625, "grad_norm": 0.4062407612800598, "learning_rate": 8.028654871074489e-05, "loss": 0.007266615517437458, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00729, "step": 177, "tokens/total": 5724672, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 80358} +{"epoch": 0.6953125, "grad_norm": 1.4019612073898315, "learning_rate": 8.004589869885986e-05, "loss": 0.0160527266561985, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.01618, "step": 178, "tokens/total": 5754384, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 80777} +{"epoch": 0.69921875, "grad_norm": 1.0894889831542969, "learning_rate": 7.980420642489674e-05, "loss": 0.016973240301012993, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01712, "step": 179, "tokens/total": 5789344, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 81250} +{"epoch": 0.703125, "grad_norm": 1.0675969123840332, "learning_rate": 7.95614819466576e-05, "loss": 0.020686637610197067, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.0209, "step": 180, "tokens/total": 5821816, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 81715} +{"epoch": 0.70703125, "grad_norm": 0.39487412571907043, "learning_rate": 7.931773536489872e-05, "loss": 0.017972204834222794, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.01813, "step": 181, "tokens/total": 5854168, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 82160} +{"epoch": 0.7109375, "grad_norm": 0.2961691915988922, "learning_rate": 7.907297682291035e-05, "loss": 0.007342329248785973, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00737, "step": 182, "tokens/total": 5886064, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 82624} +{"epoch": 0.71484375, "grad_norm": 0.7174625396728516, "learning_rate": 7.882721650609442e-05, "loss": 0.0062873405404388905, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00631, "step": 183, "tokens/total": 5921648, "tokens/train_per_sec_per_gpu": 6.76, "tokens/trainable": 83125} +{"epoch": 0.71875, "grad_norm": 0.700275719165802, "learning_rate": 7.85804646415409e-05, "loss": 0.021791720762848854, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.02203, "step": 184, "tokens/total": 5954952, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 83581} +{"epoch": 0.72265625, "grad_norm": 0.06234246864914894, "learning_rate": 7.833273149760207e-05, "loss": 0.002291259588673711, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00229, "step": 185, "tokens/total": 5985432, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 84051} +{"epoch": 0.7265625, "grad_norm": 1.12466561794281, "learning_rate": 7.808402738346527e-05, "loss": 0.027909554541110992, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.0283, "step": 186, "tokens/total": 6020664, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 84500} +{"epoch": 0.73046875, "grad_norm": 0.1173739805817604, "learning_rate": 7.783436264872382e-05, "loss": 0.002604532754048705, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00261, "step": 187, "tokens/total": 6055024, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 84986} +{"epoch": 0.734375, "grad_norm": 0.1698581725358963, "learning_rate": 7.758374768294647e-05, "loss": 0.007215807680040598, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00724, "step": 188, "tokens/total": 6088296, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 85465} +{"epoch": 0.73828125, "grad_norm": 0.17908495664596558, "learning_rate": 7.733219291524489e-05, "loss": 0.008412981405854225, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00845, "step": 189, "tokens/total": 6119840, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 85896} +{"epoch": 0.7421875, "grad_norm": 0.33263546228408813, "learning_rate": 7.707970881383977e-05, "loss": 0.005741273518651724, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.00576, "step": 190, "tokens/total": 6149880, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 86346} +{"epoch": 0.74609375, "grad_norm": 0.13297010958194733, "learning_rate": 7.682630588562518e-05, "loss": 0.0048728929832577705, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00488, "step": 191, "tokens/total": 6182904, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 86777} +{"epoch": 0.75, "grad_norm": 0.1565943956375122, "learning_rate": 7.657199467573129e-05, "loss": 0.00564972311258316, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00567, "step": 192, "tokens/total": 6214144, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 87249} +{"epoch": 0.75390625, "grad_norm": 0.4279201626777649, "learning_rate": 7.631678576708561e-05, "loss": 0.007388448342680931, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00742, "step": 193, "tokens/total": 6247072, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 87741} +{"epoch": 0.7578125, "grad_norm": 0.049396391957998276, "learning_rate": 7.606068977997255e-05, "loss": 0.0006775567890144885, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00068, "step": 194, "tokens/total": 6281112, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 88219} +{"epoch": 0.76171875, "grad_norm": 0.36552515625953674, "learning_rate": 7.580371737159148e-05, "loss": 0.0017680393066257238, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00177, "step": 195, "tokens/total": 6310200, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 88630} +{"epoch": 0.765625, "grad_norm": 0.036318980157375336, "learning_rate": 7.554587923561324e-05, "loss": 0.00032401722273789346, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00032, "step": 196, "tokens/total": 6341792, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 89068} +{"epoch": 0.76953125, "grad_norm": 0.6928939819335938, "learning_rate": 7.528718610173511e-05, "loss": 0.008479592390358448, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.00852, "step": 197, "tokens/total": 6374040, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 89489} +{"epoch": 0.7734375, "grad_norm": 0.24959585070610046, "learning_rate": 7.502764873523431e-05, "loss": 0.0017470956081524491, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00175, "step": 198, "tokens/total": 6405552, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 89928} +{"epoch": 0.77734375, "grad_norm": 2.866637706756592, "learning_rate": 7.476727793652011e-05, "loss": 0.02275856025516987, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.02302, "step": 199, "tokens/total": 6436016, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 90407} +{"epoch": 0.78125, "grad_norm": 0.36777061223983765, "learning_rate": 7.450608454068415e-05, "loss": 0.003892148844897747, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.0039, "step": 200, "tokens/total": 6468656, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 90849} +{"epoch": 0.78515625, "grad_norm": 1.36959969997406, "learning_rate": 7.424407941704987e-05, "loss": 0.018335802480578423, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.0185, "step": 201, "tokens/total": 6502728, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 91350} +{"epoch": 0.7890625, "grad_norm": 0.39073294401168823, "learning_rate": 7.398127346871986e-05, "loss": 0.002859486499801278, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00286, "step": 202, "tokens/total": 6535616, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 91805} +{"epoch": 0.79296875, "grad_norm": 0.006292284466326237, "learning_rate": 7.371767763212238e-05, "loss": 9.585910447640345e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.0001, "step": 203, "tokens/total": 6567088, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 92258} +{"epoch": 0.796875, "grad_norm": 1.807627558708191, "learning_rate": 7.345330287655617e-05, "loss": 0.004903852473944426, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00492, "step": 204, "tokens/total": 6600808, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 92721} +{"epoch": 0.80078125, "grad_norm": 1.0911157131195068, "learning_rate": 7.31881602037339e-05, "loss": 0.01461090613156557, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.01472, "step": 205, "tokens/total": 6631384, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 93160} +{"epoch": 0.8046875, "grad_norm": 0.019629742950201035, "learning_rate": 7.29222606473245e-05, "loss": 0.00020989631593693048, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00021, "step": 206, "tokens/total": 6661888, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 93605} +{"epoch": 0.80859375, "grad_norm": 0.7598876953125, "learning_rate": 7.265561527249383e-05, "loss": 0.0052677663043141365, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00528, "step": 207, "tokens/total": 6693456, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 94041} +{"epoch": 0.8125, "grad_norm": 1.8005170822143555, "learning_rate": 7.238823517544436e-05, "loss": 0.023488536477088928, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.02377, "step": 208, "tokens/total": 6727840, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 94512} +{"epoch": 0.81640625, "grad_norm": 0.05144453048706055, "learning_rate": 7.212013148295333e-05, "loss": 0.0007824697531759739, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00078, "step": 209, "tokens/total": 6761704, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 94994} +{"epoch": 0.8203125, "grad_norm": 1.12502121925354, "learning_rate": 7.185131535190975e-05, "loss": 0.0063499705865979195, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00637, "step": 210, "tokens/total": 6792648, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 95503} +{"epoch": 0.82421875, "grad_norm": 0.7333378195762634, "learning_rate": 7.158179796885005e-05, "loss": 0.00774614280089736, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00778, "step": 211, "tokens/total": 6824424, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 95954} +{"epoch": 0.828125, "grad_norm": 0.4632634222507477, "learning_rate": 7.131159054949273e-05, "loss": 0.019176695495843887, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.01936, "step": 212, "tokens/total": 6861216, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 96428} +{"epoch": 0.83203125, "grad_norm": 0.1943177729845047, "learning_rate": 7.104070433827139e-05, "loss": 0.002533521968871355, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00254, "step": 213, "tokens/total": 6892704, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 96828} +{"epoch": 0.8359375, "grad_norm": 0.12952889502048492, "learning_rate": 7.076915060786705e-05, "loss": 0.0014334805309772491, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.00143, "step": 214, "tokens/total": 6924816, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 97270} +{"epoch": 0.83984375, "grad_norm": 0.21842855215072632, "learning_rate": 7.049694065873882e-05, "loss": 0.00397136528044939, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00398, "step": 215, "tokens/total": 6957536, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 97727} +{"epoch": 0.84375, "grad_norm": 0.4380325675010681, "learning_rate": 7.022408581865382e-05, "loss": 0.0068766470067203045, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.0069, "step": 216, "tokens/total": 6991448, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 98201} +{"epoch": 0.84765625, "grad_norm": 0.08911324292421341, "learning_rate": 6.99505974422157e-05, "loss": 0.0020618478301912546, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00206, "step": 217, "tokens/total": 7023712, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 98674} +{"epoch": 0.8515625, "grad_norm": 0.09921909868717194, "learning_rate": 6.967648691039213e-05, "loss": 0.0011338479816913605, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00113, "step": 218, "tokens/total": 7055040, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 99108} +{"epoch": 0.85546875, "grad_norm": 0.3102944791316986, "learning_rate": 6.940176563004123e-05, "loss": 0.0043716467916965485, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00438, "step": 219, "tokens/total": 7084944, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 99540} +{"epoch": 0.859375, "grad_norm": 0.3667425215244293, "learning_rate": 6.912644503343682e-05, "loss": 0.0061784787103533745, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.0062, "step": 220, "tokens/total": 7119264, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 99978} +{"epoch": 0.86328125, "grad_norm": 0.4110425114631653, "learning_rate": 6.885053657779273e-05, "loss": 0.010892445221543312, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.01095, "step": 221, "tokens/total": 7150848, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 100450} +{"epoch": 0.8671875, "grad_norm": 0.04783326014876366, "learning_rate": 6.857405174478604e-05, "loss": 0.0008020481327548623, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.0008, "step": 222, "tokens/total": 7185176, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 100903} +{"epoch": 0.87109375, "grad_norm": 0.8444886207580566, "learning_rate": 6.82970020400792e-05, "loss": 0.007698482368141413, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00773, "step": 223, "tokens/total": 7219024, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 101351} +{"epoch": 0.875, "grad_norm": 0.22972355782985687, "learning_rate": 6.801939899284132e-05, "loss": 0.004061855375766754, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00407, "step": 224, "tokens/total": 7253816, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 101857} +{"epoch": 0.87890625, "grad_norm": 0.37754324078559875, "learning_rate": 6.774125415526827e-05, "loss": 0.008495159447193146, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00853, "step": 225, "tokens/total": 7286344, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 102328} +{"epoch": 0.8828125, "grad_norm": 0.20592287182807922, "learning_rate": 6.746257910210214e-05, "loss": 0.011858574114739895, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.01193, "step": 226, "tokens/total": 7317952, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 102779} +{"epoch": 0.88671875, "grad_norm": 0.12140487134456635, "learning_rate": 6.718338543014937e-05, "loss": 0.002001130487769842, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.002, "step": 227, "tokens/total": 7347336, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 103228} +{"epoch": 0.890625, "grad_norm": 0.23542344570159912, "learning_rate": 6.69036847577983e-05, "loss": 0.0033693413715809584, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00338, "step": 228, "tokens/total": 7377592, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 103684} +{"epoch": 0.89453125, "grad_norm": 0.19475284218788147, "learning_rate": 6.662348872453553e-05, "loss": 0.003997041378170252, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00401, "step": 229, "tokens/total": 7410184, "tokens/train_per_sec_per_gpu": 10.55, "tokens/trainable": 104145} +{"epoch": 0.8984375, "grad_norm": 0.01827074959874153, "learning_rate": 6.63428089904618e-05, "loss": 0.0004425144288688898, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00044, "step": 230, "tokens/total": 7443984, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 104598} +{"epoch": 0.90234375, "grad_norm": 0.3382999002933502, "learning_rate": 6.60616572358065e-05, "loss": 0.009789588861167431, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00984, "step": 231, "tokens/total": 7472480, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 104998} +{"epoch": 0.90625, "grad_norm": 0.40748924016952515, "learning_rate": 6.578004516044172e-05, "loss": 0.0061233132146298885, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00614, "step": 232, "tokens/total": 7505992, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 105465} +{"epoch": 0.91015625, "grad_norm": 0.058823537081480026, "learning_rate": 6.549798448339548e-05, "loss": 0.0005694838473573327, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00057, "step": 233, "tokens/total": 7541168, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 105928} +{"epoch": 0.9140625, "grad_norm": 0.07752551138401031, "learning_rate": 6.521548694236384e-05, "loss": 0.0008509896579198539, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00085, "step": 234, "tokens/total": 7574208, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 106365} +{"epoch": 0.91796875, "grad_norm": 0.06358706206083298, "learning_rate": 6.493256429322259e-05, "loss": 0.001334058353677392, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00133, "step": 235, "tokens/total": 7607016, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 106796} +{"epoch": 0.921875, "grad_norm": 0.027625197544693947, "learning_rate": 6.464922830953799e-05, "loss": 0.000506084761582315, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00051, "step": 236, "tokens/total": 7641744, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 107227} +{"epoch": 0.92578125, "grad_norm": 0.2130367010831833, "learning_rate": 6.436549078207688e-05, "loss": 0.0019384513143450022, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00194, "step": 237, "tokens/total": 7673096, "tokens/train_per_sec_per_gpu": 10.17, "tokens/trainable": 107687} +{"epoch": 0.9296875, "grad_norm": 0.04757566377520561, "learning_rate": 6.408136351831592e-05, "loss": 0.0005409896839410067, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00054, "step": 238, "tokens/total": 7705344, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 108127} +{"epoch": 0.93359375, "grad_norm": 0.03447839617729187, "learning_rate": 6.379685834195036e-05, "loss": 0.00041191541822627187, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00041, "step": 239, "tokens/total": 7740240, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 108633} +{"epoch": 0.9375, "grad_norm": 0.07203508168458939, "learning_rate": 6.351198709240186e-05, "loss": 0.0011779469205066562, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00118, "step": 240, "tokens/total": 7775392, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 109085} +{"epoch": 0.94140625, "grad_norm": 0.3097190260887146, "learning_rate": 6.32267616243259e-05, "loss": 0.006180963944643736, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.0062, "step": 241, "tokens/total": 7807296, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 109548} +{"epoch": 0.9453125, "grad_norm": 0.2322739064693451, "learning_rate": 6.294119380711849e-05, "loss": 0.002620828803628683, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00262, "step": 242, "tokens/total": 7838200, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 109975} +{"epoch": 0.94921875, "grad_norm": 0.9490498900413513, "learning_rate": 6.265529552442209e-05, "loss": 0.013187190517783165, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01327, "step": 243, "tokens/total": 7870936, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 110436} +{"epoch": 0.953125, "grad_norm": 0.04102562367916107, "learning_rate": 6.236907867363127e-05, "loss": 0.000247783464146778, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00025, "step": 244, "tokens/total": 7902512, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 110882} +{"epoch": 0.95703125, "grad_norm": 0.01411413587629795, "learning_rate": 6.208255516539749e-05, "loss": 0.0002416871429886669, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00024, "step": 245, "tokens/total": 7933224, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 111355} +{"epoch": 0.9609375, "grad_norm": 0.24223046004772186, "learning_rate": 6.179573692313344e-05, "loss": 0.008175211027264595, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00821, "step": 246, "tokens/total": 7968456, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 111803} +{"epoch": 0.96484375, "grad_norm": 0.32318001985549927, "learning_rate": 6.150863588251694e-05, "loss": 0.0029268586076796055, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00293, "step": 247, "tokens/total": 8003592, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 112270} +{"epoch": 0.96875, "grad_norm": 0.2235661894083023, "learning_rate": 6.122126399099419e-05, "loss": 0.002423725090920925, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00243, "step": 248, "tokens/total": 8040048, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 112778} +{"epoch": 0.97265625, "grad_norm": 0.43262478709220886, "learning_rate": 6.0933633207282615e-05, "loss": 0.012419785372912884, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.0125, "step": 249, "tokens/total": 8074000, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 113284} +{"epoch": 0.9765625, "grad_norm": 0.11046585440635681, "learning_rate": 6.064575550087316e-05, "loss": 0.0008808902930468321, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00088, "step": 250, "tokens/total": 8108304, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 113722} +{"epoch": 0.98046875, "grad_norm": 0.25166067481040955, "learning_rate": 6.0357642851532245e-05, "loss": 0.0022622058168053627, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00226, "step": 251, "tokens/total": 8143464, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 114213} +{"epoch": 0.984375, "grad_norm": 0.027573266997933388, "learning_rate": 6.0069307248803294e-05, "loss": 0.00028019185992889106, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00028, "step": 252, "tokens/total": 8175576, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 114634} +{"epoch": 0.98828125, "grad_norm": 0.2396245002746582, "learning_rate": 5.9780760691507635e-05, "loss": 0.003622143529355526, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00363, "step": 253, "tokens/total": 8205528, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 115066} +{"epoch": 0.9921875, "grad_norm": 0.22304584085941315, "learning_rate": 5.9492015187245334e-05, "loss": 0.001993746031075716, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.002, "step": 254, "tokens/total": 8239424, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 115520} +{"epoch": 0.99609375, "grad_norm": 0.14115314185619354, "learning_rate": 5.920308275189541e-05, "loss": 0.001680104061961174, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00168, "step": 255, "tokens/total": 8270392, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 115921} +{"epoch": 1.0, "grad_norm": 0.21341633796691895, "learning_rate": 5.8913975409115874e-05, "loss": 0.0021332968026399612, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00214, "step": 256, "tokens/total": 8302096, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 116384} +{"epoch": 1.00390625, "grad_norm": 0.847640335559845, "learning_rate": 5.8624705189843395e-05, "loss": 0.0047211977653205395, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00473, "step": 257, "tokens/total": 8337064, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 116871} +{"epoch": 1.0078125, "grad_norm": 0.09765225648880005, "learning_rate": 5.833528413179249e-05, "loss": 0.0004514572210609913, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00045, "step": 258, "tokens/total": 8372840, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 117369} +{"epoch": 1.01171875, "grad_norm": 0.09552741050720215, "learning_rate": 5.80457242789548e-05, "loss": 0.0015115938149392605, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00151, "step": 259, "tokens/total": 8409728, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 117866} +{"epoch": 1.015625, "grad_norm": 0.25673907995224, "learning_rate": 5.77560376810977e-05, "loss": 0.002123200800269842, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.00213, "step": 260, "tokens/total": 8442296, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 118330} +{"epoch": 1.01953125, "grad_norm": 0.015278138220310211, "learning_rate": 5.7466236393263005e-05, "loss": 0.00028761799330823123, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00029, "step": 261, "tokens/total": 8474568, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 118776} +{"epoch": 1.0234375, "grad_norm": 0.04481669142842293, "learning_rate": 5.717633247526522e-05, "loss": 0.0006861239671707153, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00069, "step": 262, "tokens/total": 8505520, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 119211} +{"epoch": 1.02734375, "grad_norm": 0.2613995373249054, "learning_rate": 5.688633799118971e-05, "loss": 0.0033120824955403805, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00332, "step": 263, "tokens/total": 8535064, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 119616} +{"epoch": 1.03125, "grad_norm": 0.21862144768238068, "learning_rate": 5.659626500889066e-05, "loss": 0.0030042428988963366, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00301, "step": 264, "tokens/total": 8563600, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 120086} +{"epoch": 1.03515625, "grad_norm": 0.7571537494659424, "learning_rate": 5.6306125599488905e-05, "loss": 0.009924478828907013, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00997, "step": 265, "tokens/total": 8592648, "tokens/train_per_sec_per_gpu": 9.88, "tokens/trainable": 120525} +{"epoch": 1.0390625, "grad_norm": 0.19929669797420502, "learning_rate": 5.601593183686955e-05, "loss": 0.0033775679767131805, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00338, "step": 266, "tokens/total": 8627328, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 121001} +{"epoch": 1.04296875, "grad_norm": 0.06862270832061768, "learning_rate": 5.572569579717961e-05, "loss": 0.0008159371209330857, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00082, "step": 267, "tokens/total": 8659760, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 121465} +{"epoch": 1.046875, "grad_norm": 0.026564370840787888, "learning_rate": 5.543542955832538e-05, "loss": 0.00012559167225845158, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00013, "step": 268, "tokens/total": 8696968, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 121908} +{"epoch": 1.05078125, "grad_norm": 0.23632915318012238, "learning_rate": 5.514514519946986e-05, "loss": 0.002108483575284481, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00211, "step": 269, "tokens/total": 8735392, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 122406} +{"epoch": 1.0546875, "grad_norm": 0.05617673322558403, "learning_rate": 5.485485480053015e-05, "loss": 0.00039701079367659986, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.0004, "step": 270, "tokens/total": 8769840, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 122812} +{"epoch": 1.05859375, "grad_norm": 0.009457884356379509, "learning_rate": 5.4564570441674645e-05, "loss": 0.00012225423415657133, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00012, "step": 271, "tokens/total": 8800648, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 123252} +{"epoch": 1.0625, "grad_norm": 0.4692811071872711, "learning_rate": 5.42743042028204e-05, "loss": 0.008894539438188076, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00893, "step": 272, "tokens/total": 8832480, "tokens/train_per_sec_per_gpu": 6.89, "tokens/trainable": 123692} +{"epoch": 1.06640625, "grad_norm": 0.013112942688167095, "learning_rate": 5.3984068163130464e-05, "loss": 0.00016197221702896059, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00016, "step": 273, "tokens/total": 8864840, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 124162} +{"epoch": 1.0703125, "grad_norm": 0.47857409715652466, "learning_rate": 5.369387440051111e-05, "loss": 0.002763098804280162, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00277, "step": 274, "tokens/total": 8897272, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 124625} +{"epoch": 1.07421875, "grad_norm": 0.008661556988954544, "learning_rate": 5.340373499110935e-05, "loss": 9.109060920309275e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00009, "step": 275, "tokens/total": 8930688, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 125119} +{"epoch": 1.078125, "grad_norm": 0.35366737842559814, "learning_rate": 5.3113662008810304e-05, "loss": 0.0052338396199047565, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00525, "step": 276, "tokens/total": 8967216, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 125601} +{"epoch": 1.08203125, "grad_norm": 0.021444087848067284, "learning_rate": 5.282366752473479e-05, "loss": 0.00017079163808375597, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00017, "step": 277, "tokens/total": 9001368, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 126075} +{"epoch": 1.0859375, "grad_norm": 0.02260698936879635, "learning_rate": 5.2533763606737005e-05, "loss": 0.0002920105471275747, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00029, "step": 278, "tokens/total": 9036808, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 126491} +{"epoch": 1.08984375, "grad_norm": 0.013469593599438667, "learning_rate": 5.224396231890232e-05, "loss": 9.442596638109535e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00009, "step": 279, "tokens/total": 9067840, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 126940} +{"epoch": 1.09375, "grad_norm": 0.054189082235097885, "learning_rate": 5.195427572104522e-05, "loss": 0.0006044308538548648, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.0006, "step": 280, "tokens/total": 9098936, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 127376} +{"epoch": 1.09765625, "grad_norm": 0.0219765305519104, "learning_rate": 5.166471586820751e-05, "loss": 0.00022247989545576274, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00022, "step": 281, "tokens/total": 9133264, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 127841} +{"epoch": 1.1015625, "grad_norm": 0.3126583993434906, "learning_rate": 5.1375294810156615e-05, "loss": 0.009371907450258732, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00942, "step": 282, "tokens/total": 9165008, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 128288} +{"epoch": 1.10546875, "grad_norm": 0.022368911653757095, "learning_rate": 5.1086024590884144e-05, "loss": 0.00021839566761627793, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00022, "step": 283, "tokens/total": 9197952, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 128737} +{"epoch": 1.109375, "grad_norm": 0.07086675614118576, "learning_rate": 5.079691724810461e-05, "loss": 0.0009468976641073823, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00095, "step": 284, "tokens/total": 9232576, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 129172} +{"epoch": 1.11328125, "grad_norm": 0.17183347046375275, "learning_rate": 5.0507984812754684e-05, "loss": 0.0011287083616480231, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00113, "step": 285, "tokens/total": 9264568, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 129669} +{"epoch": 1.1171875, "grad_norm": 0.13570959866046906, "learning_rate": 5.021923930849237e-05, "loss": 0.0015209023840725422, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00152, "step": 286, "tokens/total": 9296544, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 130155} +{"epoch": 1.12109375, "grad_norm": 0.007033331319689751, "learning_rate": 4.99306927511967e-05, "loss": 8.011833415366709e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00008, "step": 287, "tokens/total": 9327936, "tokens/train_per_sec_per_gpu": 10.46, "tokens/trainable": 130595} +{"epoch": 1.125, "grad_norm": 0.08613824844360352, "learning_rate": 4.964235714846775e-05, "loss": 0.0003713329788297415, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00037, "step": 288, "tokens/total": 9357984, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 131015} +{"epoch": 1.12890625, "grad_norm": 0.029945088550448418, "learning_rate": 4.9354244499126866e-05, "loss": 0.00035140427644364536, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00035, "step": 289, "tokens/total": 9388664, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 131481} +{"epoch": 1.1328125, "grad_norm": 0.1466934233903885, "learning_rate": 4.90663667927174e-05, "loss": 0.0011842178646475077, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00118, "step": 290, "tokens/total": 9420504, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 131930} +{"epoch": 1.13671875, "grad_norm": 0.13517360389232635, "learning_rate": 4.877873600900581e-05, "loss": 0.001233975519426167, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00123, "step": 291, "tokens/total": 9452664, "tokens/train_per_sec_per_gpu": 6.24, "tokens/trainable": 132388} +{"epoch": 1.140625, "grad_norm": 0.03252067789435387, "learning_rate": 4.849136411748306e-05, "loss": 0.00014405159163288772, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00014, "step": 292, "tokens/total": 9488896, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 132860} +{"epoch": 1.14453125, "grad_norm": 0.32541415095329285, "learning_rate": 4.8204263076866574e-05, "loss": 0.005052025895565748, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00506, "step": 293, "tokens/total": 9522112, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 133360} +{"epoch": 1.1484375, "grad_norm": 0.014466228894889355, "learning_rate": 4.791744483460251e-05, "loss": 7.572740287287161e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00008, "step": 294, "tokens/total": 9555632, "tokens/train_per_sec_per_gpu": 6.6, "tokens/trainable": 133793} +{"epoch": 1.15234375, "grad_norm": 0.0025463944766670465, "learning_rate": 4.7630921326368736e-05, "loss": 2.7098471036879346e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00003, "step": 295, "tokens/total": 9590904, "tokens/train_per_sec_per_gpu": 6.44, "tokens/trainable": 134218} +{"epoch": 1.15625, "grad_norm": 0.5030266046524048, "learning_rate": 4.7344704475577916e-05, "loss": 0.0060522183775901794, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00607, "step": 296, "tokens/total": 9620288, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 134654} +{"epoch": 1.16015625, "grad_norm": 0.07132592052221298, "learning_rate": 4.705880619288153e-05, "loss": 0.0005382879171520472, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00054, "step": 297, "tokens/total": 9651984, "tokens/train_per_sec_per_gpu": 7.11, "tokens/trainable": 135056} +{"epoch": 1.1640625, "grad_norm": 0.31259724497795105, "learning_rate": 4.677323837567412e-05, "loss": 0.0022502229548990726, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00225, "step": 298, "tokens/total": 9681368, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 135483} +{"epoch": 1.16796875, "grad_norm": 0.22073085606098175, "learning_rate": 4.6488012907598146e-05, "loss": 0.0001264017482753843, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00013, "step": 299, "tokens/total": 9712872, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 135957} +{"epoch": 1.171875, "grad_norm": 1.7412011623382568, "learning_rate": 4.620314165804964e-05, "loss": 0.0019509458215907216, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00195, "step": 300, "tokens/total": 9745184, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 136430} +{"epoch": 1.17578125, "grad_norm": 0.02497597597539425, "learning_rate": 4.591863648168407e-05, "loss": 0.00019092178263235837, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00019, "step": 301, "tokens/total": 9778928, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 136896} +{"epoch": 1.1796875, "grad_norm": 0.16146856546401978, "learning_rate": 4.5634509217923135e-05, "loss": 0.0017394438618794084, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00174, "step": 302, "tokens/total": 9810752, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 137331} +{"epoch": 1.18359375, "grad_norm": 0.30532118678092957, "learning_rate": 4.535077169046201e-05, "loss": 0.001794431358575821, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.0018, "step": 303, "tokens/total": 9843672, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 137768} +{"epoch": 1.1875, "grad_norm": 0.5057659149169922, "learning_rate": 4.506743570677743e-05, "loss": 0.0034931045956909657, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.0035, "step": 304, "tokens/total": 9873752, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 138200} +{"epoch": 1.19140625, "grad_norm": 0.058327816426754, "learning_rate": 4.478451305763618e-05, "loss": 0.0002982247096952051, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.0003, "step": 305, "tokens/total": 9906272, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 138660} +{"epoch": 1.1953125, "grad_norm": 0.0010027947137132287, "learning_rate": 4.450201551660454e-05, "loss": 1.0933901648968458e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00001, "step": 306, "tokens/total": 9936296, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 139092} +{"epoch": 1.19921875, "grad_norm": 0.7508344650268555, "learning_rate": 4.4219954839558276e-05, "loss": 0.004291347693651915, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0043, "step": 307, "tokens/total": 9967720, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 139533} +{"epoch": 1.203125, "grad_norm": 0.00759107805788517, "learning_rate": 4.393834276419352e-05, "loss": 7.202685810625553e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.00007, "step": 308, "tokens/total": 9997176, "tokens/train_per_sec_per_gpu": 10.82, "tokens/trainable": 140020} +{"epoch": 1.20703125, "grad_norm": 0.017406325787305832, "learning_rate": 4.36571910095382e-05, "loss": 0.00015312190225813538, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00015, "step": 309, "tokens/total": 10031872, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 140465} +{"epoch": 1.2109375, "grad_norm": 0.039860498160123825, "learning_rate": 4.337651127546448e-05, "loss": 0.00025978047051467, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00026, "step": 310, "tokens/total": 10065240, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 140915} +{"epoch": 1.21484375, "grad_norm": 0.03213198855519295, "learning_rate": 4.3096315242201736e-05, "loss": 0.00028655401547439396, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00029, "step": 311, "tokens/total": 10096640, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 141350} +{"epoch": 1.21875, "grad_norm": 0.0005637833382934332, "learning_rate": 4.2816614569850635e-05, "loss": 1.0223559002042748e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00001, "step": 312, "tokens/total": 10130192, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 141789} +{"epoch": 1.22265625, "grad_norm": 0.11343056708574295, "learning_rate": 4.2537420897897864e-05, "loss": 0.0005871893372386694, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00059, "step": 313, "tokens/total": 10164248, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 142266} +{"epoch": 1.2265625, "grad_norm": 0.006020143162459135, "learning_rate": 4.225874584473174e-05, "loss": 4.853140490013175e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00005, "step": 314, "tokens/total": 10194664, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 142704} +{"epoch": 1.23046875, "grad_norm": 0.16482137143611908, "learning_rate": 4.19806010071587e-05, "loss": 0.0006413726950995624, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00064, "step": 315, "tokens/total": 10227768, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 143168} +{"epoch": 1.234375, "grad_norm": 0.03929819166660309, "learning_rate": 4.170299795992081e-05, "loss": 0.0003845624451059848, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00038, "step": 316, "tokens/total": 10260464, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 143615} +{"epoch": 1.23828125, "grad_norm": 0.7124431133270264, "learning_rate": 4.142594825521398e-05, "loss": 0.006129699293524027, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00615, "step": 317, "tokens/total": 10297528, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 144096} +{"epoch": 1.2421875, "grad_norm": 0.0019236798398196697, "learning_rate": 4.114946342220728e-05, "loss": 1.741759479045868e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00002, "step": 318, "tokens/total": 10331448, "tokens/train_per_sec_per_gpu": 7.0, "tokens/trainable": 144543} +{"epoch": 1.24609375, "grad_norm": 0.009873388335108757, "learning_rate": 4.087355496656321e-05, "loss": 8.302739297505468e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00008, "step": 319, "tokens/total": 10363152, "tokens/train_per_sec_per_gpu": 10.19, "tokens/trainable": 145016} +{"epoch": 1.25, "grad_norm": 0.10226759314537048, "learning_rate": 4.05982343699588e-05, "loss": 0.0006503399927169085, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00065, "step": 320, "tokens/total": 10397832, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 145511} +{"epoch": 1.25390625, "grad_norm": 0.0627579391002655, "learning_rate": 4.0323513089607876e-05, "loss": 0.0006250463193282485, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00063, "step": 321, "tokens/total": 10431416, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 145978} +{"epoch": 1.2578125, "grad_norm": 0.10126078873872757, "learning_rate": 4.004940255778431e-05, "loss": 0.001135424361564219, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00114, "step": 322, "tokens/total": 10467384, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 146422} +{"epoch": 1.26171875, "grad_norm": 0.009869217872619629, "learning_rate": 3.977591418134619e-05, "loss": 0.000110771092295181, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00011, "step": 323, "tokens/total": 10500792, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 146863} +{"epoch": 1.265625, "grad_norm": 0.06345292925834656, "learning_rate": 3.95030593412612e-05, "loss": 0.0006929629598744214, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00069, "step": 324, "tokens/total": 10532592, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 147286} +{"epoch": 1.26953125, "grad_norm": 0.004015319515019655, "learning_rate": 3.923084939213296e-05, "loss": 3.705283234012313e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00004, "step": 325, "tokens/total": 10562392, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 147708} +{"epoch": 1.2734375, "grad_norm": 0.6207872033119202, "learning_rate": 3.895929566172861e-05, "loss": 0.004071317613124847, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00408, "step": 326, "tokens/total": 10595416, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 148172} +{"epoch": 1.27734375, "grad_norm": 0.03559695929288864, "learning_rate": 3.868840945050728e-05, "loss": 0.00028544256929308176, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00029, "step": 327, "tokens/total": 10630504, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 148604} +{"epoch": 1.28125, "grad_norm": 0.24147193133831024, "learning_rate": 3.841820203114995e-05, "loss": 0.001600753515958786, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.0016, "step": 328, "tokens/total": 10661632, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 149051} +{"epoch": 1.28515625, "grad_norm": 0.09421749413013458, "learning_rate": 3.814868464809027e-05, "loss": 0.00013797350402455777, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00014, "step": 329, "tokens/total": 10691544, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 149453} +{"epoch": 1.2890625, "grad_norm": 0.20627810060977936, "learning_rate": 3.787986851704667e-05, "loss": 0.000999743235297501, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.001, "step": 330, "tokens/total": 10724768, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 149883} +{"epoch": 1.29296875, "grad_norm": 0.05720088258385658, "learning_rate": 3.7611764824555654e-05, "loss": 0.0004321803280618042, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00043, "step": 331, "tokens/total": 10756008, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 150357} +{"epoch": 1.296875, "grad_norm": 0.16884225606918335, "learning_rate": 3.734438472750619e-05, "loss": 0.0009627511026337743, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00096, "step": 332, "tokens/total": 10793152, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 150828} +{"epoch": 1.30078125, "grad_norm": 0.02390812709927559, "learning_rate": 3.707773935267552e-05, "loss": 9.21185128390789e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00009, "step": 333, "tokens/total": 10827624, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 151314} +{"epoch": 1.3046875, "grad_norm": 0.005645441822707653, "learning_rate": 3.68118397962661e-05, "loss": 3.999929322162643e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00004, "step": 334, "tokens/total": 10859712, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 151722} +{"epoch": 1.30859375, "grad_norm": 0.18057233095169067, "learning_rate": 3.654669712344384e-05, "loss": 0.0005824364488944411, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00058, "step": 335, "tokens/total": 10888792, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 152178} +{"epoch": 1.3125, "grad_norm": 0.00774751603603363, "learning_rate": 3.628232236787763e-05, "loss": 4.9436261178925633e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00005, "step": 336, "tokens/total": 10922968, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 152617} +{"epoch": 1.31640625, "grad_norm": 0.037371374666690826, "learning_rate": 3.6018726531280144e-05, "loss": 0.0001847979292506352, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00018, "step": 337, "tokens/total": 10958000, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 153108} +{"epoch": 1.3203125, "grad_norm": 0.16463355720043182, "learning_rate": 3.575592058295017e-05, "loss": 0.0008722989005036652, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00087, "step": 338, "tokens/total": 10989872, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 153589} +{"epoch": 1.32421875, "grad_norm": 0.00881918240338564, "learning_rate": 3.549391545931585e-05, "loss": 3.997977182734758e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00004, "step": 339, "tokens/total": 11020784, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 154049} +{"epoch": 1.328125, "grad_norm": 0.009240211918950081, "learning_rate": 3.5232722063479914e-05, "loss": 5.1087659812765196e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00005, "step": 340, "tokens/total": 11053240, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 154498} +{"epoch": 1.33203125, "grad_norm": 0.008867397904396057, "learning_rate": 3.49723512647657e-05, "loss": 8.479662210447714e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00008, "step": 341, "tokens/total": 11089040, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 154998} +{"epoch": 1.3359375, "grad_norm": 0.14443467557430267, "learning_rate": 3.471281389826491e-05, "loss": 0.00046477673458866775, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00046, "step": 342, "tokens/total": 11119672, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 155403} +{"epoch": 1.33984375, "grad_norm": 0.7290111184120178, "learning_rate": 3.4454120764386764e-05, "loss": 0.01981164701282978, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.02001, "step": 343, "tokens/total": 11155040, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 155858} +{"epoch": 1.34375, "grad_norm": 0.0338488332927227, "learning_rate": 3.4196282628408526e-05, "loss": 9.715838677948341e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.0001, "step": 344, "tokens/total": 11185952, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 156303} +{"epoch": 1.34765625, "grad_norm": 0.010136299766600132, "learning_rate": 3.3939310220027456e-05, "loss": 5.151806544745341e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00005, "step": 345, "tokens/total": 11219272, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 156798} +{"epoch": 1.3515625, "grad_norm": 0.08945762366056442, "learning_rate": 3.3683214232914404e-05, "loss": 0.000285696325590834, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00029, "step": 346, "tokens/total": 11253424, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 157271} +{"epoch": 1.35546875, "grad_norm": 0.003890440333634615, "learning_rate": 3.342800532426873e-05, "loss": 2.37276399275288e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00002, "step": 347, "tokens/total": 11286216, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 157737} +{"epoch": 1.359375, "grad_norm": 0.06738550215959549, "learning_rate": 3.317369411437484e-05, "loss": 0.0004961176309734583, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.0005, "step": 348, "tokens/total": 11319480, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 158205} +{"epoch": 1.36328125, "grad_norm": 0.004641884937882423, "learning_rate": 3.292029118616024e-05, "loss": 3.204824315616861e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00003, "step": 349, "tokens/total": 11350472, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 158628} +{"epoch": 1.3671875, "grad_norm": 0.005976130720227957, "learning_rate": 3.266780708475511e-05, "loss": 6.904828478582203e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00007, "step": 350, "tokens/total": 11383088, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 159106} +{"epoch": 1.37109375, "grad_norm": 0.016105975955724716, "learning_rate": 3.241625231705354e-05, "loss": 0.00012073642574250698, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00012, "step": 351, "tokens/total": 11418392, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 159565} +{"epoch": 1.375, "grad_norm": 0.011442044749855995, "learning_rate": 3.216563735127618e-05, "loss": 8.038292435230687e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00008, "step": 352, "tokens/total": 11449784, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 160042} +{"epoch": 1.37890625, "grad_norm": 0.40536075830459595, "learning_rate": 3.191597261653475e-05, "loss": 0.002259016502648592, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00226, "step": 353, "tokens/total": 11484240, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 160502} +{"epoch": 1.3828125, "grad_norm": 0.017010357230901718, "learning_rate": 3.166726850239794e-05, "loss": 0.00010349843796575442, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.0001, "step": 354, "tokens/total": 11513360, "tokens/train_per_sec_per_gpu": 11.03, "tokens/trainable": 160950} +{"epoch": 1.38671875, "grad_norm": 0.003667799523100257, "learning_rate": 3.141953535845912e-05, "loss": 4.5658438466489315e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00005, "step": 355, "tokens/total": 11543112, "tokens/train_per_sec_per_gpu": 10.29, "tokens/trainable": 161400} +{"epoch": 1.390625, "grad_norm": 0.03575177118182182, "learning_rate": 3.11727834939056e-05, "loss": 0.00026124573196284473, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00026, "step": 356, "tokens/total": 11573464, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 161863} +{"epoch": 1.39453125, "grad_norm": 0.005828642286360264, "learning_rate": 3.092702317708967e-05, "loss": 5.689292447641492e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00006, "step": 357, "tokens/total": 11604896, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 162319} +{"epoch": 1.3984375, "grad_norm": 0.0021195842418819666, "learning_rate": 3.0682264635101276e-05, "loss": 3.3787626307457685e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00003, "step": 358, "tokens/total": 11636368, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 162771} +{"epoch": 1.40234375, "grad_norm": 0.6020651459693909, "learning_rate": 3.0438518053342407e-05, "loss": 0.005935001652687788, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00595, "step": 359, "tokens/total": 11670216, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 163243} +{"epoch": 1.40625, "grad_norm": 0.033591803163290024, "learning_rate": 3.0195793575103266e-05, "loss": 0.00024454196682199836, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00024, "step": 360, "tokens/total": 11707816, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 163707} +{"epoch": 1.41015625, "grad_norm": 0.1319909393787384, "learning_rate": 2.9954101301140146e-05, "loss": 0.0005781518993899226, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00058, "step": 361, "tokens/total": 11742584, "tokens/train_per_sec_per_gpu": 6.93, "tokens/trainable": 164157} +{"epoch": 1.4140625, "grad_norm": 0.005377557594329119, "learning_rate": 2.9713451289255123e-05, "loss": 5.1693557907128707e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00005, "step": 362, "tokens/total": 11770384, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 164598} +{"epoch": 1.41796875, "grad_norm": 0.05858879163861275, "learning_rate": 2.9473853553877484e-05, "loss": 0.0004855134175159037, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00049, "step": 363, "tokens/total": 11803208, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 165072} +{"epoch": 1.421875, "grad_norm": 0.02576388418674469, "learning_rate": 2.9235318065647e-05, "loss": 0.00029737615841440856, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.0003, "step": 364, "tokens/total": 11840480, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 165533} +{"epoch": 1.42578125, "grad_norm": 0.008512251079082489, "learning_rate": 2.8997854750998964e-05, "loss": 2.5093087970162742e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00003, "step": 365, "tokens/total": 11875544, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 165998} +{"epoch": 1.4296875, "grad_norm": 0.006436346098780632, "learning_rate": 2.8761473491751258e-05, "loss": 6.377081444952637e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00006, "step": 366, "tokens/total": 11907656, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 166433} +{"epoch": 1.43359375, "grad_norm": 0.13158103823661804, "learning_rate": 2.8526184124692883e-05, "loss": 0.0007208537426777184, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00072, "step": 367, "tokens/total": 11941904, "tokens/train_per_sec_per_gpu": 6.43, "tokens/trainable": 166871} +{"epoch": 1.4375, "grad_norm": 0.589269757270813, "learning_rate": 2.829199644117484e-05, "loss": 0.0036568865180015564, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00366, "step": 368, "tokens/total": 11973624, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 167281} +{"epoch": 1.44140625, "grad_norm": 0.005107713397592306, "learning_rate": 2.8058920186702553e-05, "loss": 6.211949221324176e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00006, "step": 369, "tokens/total": 12010360, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 167754} +{"epoch": 1.4453125, "grad_norm": 0.2214343249797821, "learning_rate": 2.782696506053033e-05, "loss": 0.0015694423345848918, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00157, "step": 370, "tokens/total": 12044544, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 168186} +{"epoch": 1.44921875, "grad_norm": 0.01235619094222784, "learning_rate": 2.7596140715257824e-05, "loss": 0.00011921938857994974, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00012, "step": 371, "tokens/total": 12075792, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 168651} +{"epoch": 1.453125, "grad_norm": 0.037339769303798676, "learning_rate": 2.7366456756428184e-05, "loss": 0.0004137077194172889, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00041, "step": 372, "tokens/total": 12108088, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 169108} +{"epoch": 1.45703125, "grad_norm": 0.25063031911849976, "learning_rate": 2.7137922742128486e-05, "loss": 0.0014794162707403302, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00148, "step": 373, "tokens/total": 12140888, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 169607} +{"epoch": 1.4609375, "grad_norm": 0.4746065139770508, "learning_rate": 2.691054818259188e-05, "loss": 0.0016516278265044093, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00165, "step": 374, "tokens/total": 12174560, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 170080} +{"epoch": 1.46484375, "grad_norm": 0.22100697457790375, "learning_rate": 2.6684342539801933e-05, "loss": 0.00044341027387417853, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00044, "step": 375, "tokens/total": 12204848, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 170532} +{"epoch": 1.46875, "grad_norm": 0.004810035228729248, "learning_rate": 2.645931522709877e-05, "loss": 2.1617488528136164e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00002, "step": 376, "tokens/total": 12238472, "tokens/train_per_sec_per_gpu": 10.79, "tokens/trainable": 171012} +{"epoch": 1.47265625, "grad_norm": 0.02097276970744133, "learning_rate": 2.6235475608787365e-05, "loss": 0.00013308992492966354, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00013, "step": 377, "tokens/total": 12271024, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 171473} +{"epoch": 1.4765625, "grad_norm": 0.0077109611593186855, "learning_rate": 2.6012832999747916e-05, "loss": 6.69958972139284e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00007, "step": 378, "tokens/total": 12301312, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 171920} +{"epoch": 1.48046875, "grad_norm": 0.26713746786117554, "learning_rate": 2.579139666504821e-05, "loss": 0.02530716545879841, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.02563, "step": 379, "tokens/total": 12337072, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 172366} +{"epoch": 1.484375, "grad_norm": 0.012320462614297867, "learning_rate": 2.557117581955798e-05, "loss": 0.00010919932537944987, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00011, "step": 380, "tokens/total": 12368000, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 172832} +{"epoch": 1.48828125, "grad_norm": 0.07017697393894196, "learning_rate": 2.5352179627565532e-05, "loss": 0.0005859914817847311, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00059, "step": 381, "tokens/total": 12399880, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 173272} +{"epoch": 1.4921875, "grad_norm": 0.016116879880428314, "learning_rate": 2.5134417202396277e-05, "loss": 9.65660874499008e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.0001, "step": 382, "tokens/total": 12432288, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 173679} +{"epoch": 1.49609375, "grad_norm": 0.02052406780421734, "learning_rate": 2.491789760603361e-05, "loss": 0.0002241847396362573, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00022, "step": 383, "tokens/total": 12465592, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 174109} +{"epoch": 1.5, "grad_norm": 0.0097832465544343, "learning_rate": 2.4702629848741764e-05, "loss": 6.985277286730707e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00007, "step": 384, "tokens/total": 12498200, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 174564} +{"epoch": 1.50390625, "grad_norm": 0.01451887283474207, "learning_rate": 2.4488622888690785e-05, "loss": 0.00022025102225597948, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00022, "step": 385, "tokens/total": 12530280, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 175004} +{"epoch": 1.5078125, "grad_norm": 0.0034912729170173407, "learning_rate": 2.427588563158384e-05, "loss": 4.138430449529551e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00004, "step": 386, "tokens/total": 12563360, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 175490} +{"epoch": 1.51171875, "grad_norm": 0.04676974564790726, "learning_rate": 2.406442693028651e-05, "loss": 0.0004165551217738539, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00042, "step": 387, "tokens/total": 12593032, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 175911} +{"epoch": 1.515625, "grad_norm": 0.0012408969923853874, "learning_rate": 2.3854255584458547e-05, "loss": 2.1403291611932218e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00002, "step": 388, "tokens/total": 12628272, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 176338} +{"epoch": 1.51953125, "grad_norm": 0.03757263720035553, "learning_rate": 2.3645380340187508e-05, "loss": 0.00020400373614393175, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.0002, "step": 389, "tokens/total": 12661368, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 176819} +{"epoch": 1.5234375, "grad_norm": 0.08300289511680603, "learning_rate": 2.3437809889624914e-05, "loss": 0.00047949288273230195, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.77, "memory/max_allocated (GiB)": 63.77, "ppl": 1.00048, "step": 390, "tokens/total": 12691168, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 177270} +{"epoch": 1.52734375, "grad_norm": 0.008149905130267143, "learning_rate": 2.3231552870624487e-05, "loss": 8.612433884991333e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00009, "step": 391, "tokens/total": 12721264, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 177696} +{"epoch": 1.53125, "grad_norm": 0.47654539346694946, "learning_rate": 2.3026617866382657e-05, "loss": 0.006362074054777622, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00638, "step": 392, "tokens/total": 12753640, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 178193} +{"epoch": 1.53515625, "grad_norm": 0.04522240161895752, "learning_rate": 2.2823013405081507e-05, "loss": 0.0004219037655275315, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00042, "step": 393, "tokens/total": 12787056, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 178690} +{"epoch": 1.5390625, "grad_norm": 0.001135240076109767, "learning_rate": 2.2620747959533722e-05, "loss": 1.9457313101156615e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00002, "step": 394, "tokens/total": 12819312, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 179187} +{"epoch": 1.54296875, "grad_norm": 2.1452672481536865, "learning_rate": 2.2419829946830123e-05, "loss": 0.0018271139124408364, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00183, "step": 395, "tokens/total": 12853376, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 179638} +{"epoch": 1.546875, "grad_norm": 0.05018415302038193, "learning_rate": 2.2220267727989325e-05, "loss": 0.000447621860075742, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00045, "step": 396, "tokens/total": 12882960, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 180091} +{"epoch": 1.55078125, "grad_norm": 0.11067790538072586, "learning_rate": 2.202206960760984e-05, "loss": 0.0013463926734402776, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00135, "step": 397, "tokens/total": 12917016, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 180598} +{"epoch": 1.5546875, "grad_norm": 0.1367519497871399, "learning_rate": 2.182524383352446e-05, "loss": 0.0005913832574151456, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00059, "step": 398, "tokens/total": 12950056, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 181051} +{"epoch": 1.55859375, "grad_norm": 0.3427830636501312, "learning_rate": 2.1629798596457056e-05, "loss": 0.0031194640323519707, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00312, "step": 399, "tokens/total": 12981728, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 181501} +{"epoch": 1.5625, "grad_norm": 0.5089629292488098, "learning_rate": 2.1435742029681725e-05, "loss": 0.0048480867408216, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00486, "step": 400, "tokens/total": 13011464, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 181918} +{"epoch": 1.56640625, "grad_norm": 0.07966180890798569, "learning_rate": 2.124308220868431e-05, "loss": 0.00042260377085767686, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00042, "step": 401, "tokens/total": 13044432, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 182373} +{"epoch": 1.5703125, "grad_norm": 0.009835252538323402, "learning_rate": 2.105182715082638e-05, "loss": 0.00012604915536940098, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00013, "step": 402, "tokens/total": 13075608, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 182850} +{"epoch": 1.57421875, "grad_norm": 0.009178237058222294, "learning_rate": 2.0861984815011552e-05, "loss": 4.992949470761232e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00005, "step": 403, "tokens/total": 13107352, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 183281} +{"epoch": 1.578125, "grad_norm": 0.008871038444340229, "learning_rate": 2.0673563101354323e-05, "loss": 6.807709723943844e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00007, "step": 404, "tokens/total": 13140736, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 183711} +{"epoch": 1.58203125, "grad_norm": 0.001641217153519392, "learning_rate": 2.0486569850851317e-05, "loss": 2.9365317459451035e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00003, "step": 405, "tokens/total": 13174040, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 184174} +{"epoch": 1.5859375, "grad_norm": 0.004949682857841253, "learning_rate": 2.0301012845054956e-05, "loss": 8.461985271424055e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00008, "step": 406, "tokens/total": 13207232, "tokens/train_per_sec_per_gpu": 6.68, "tokens/trainable": 184589} +{"epoch": 1.58984375, "grad_norm": 0.003733002580702305, "learning_rate": 2.011689980574966e-05, "loss": 4.410841575008817e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00004, "step": 407, "tokens/total": 13241160, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 185009} +{"epoch": 1.59375, "grad_norm": 0.0036379597149789333, "learning_rate": 1.993423839463052e-05, "loss": 6.299841334111989e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00006, "step": 408, "tokens/total": 13273104, "tokens/train_per_sec_per_gpu": 10.47, "tokens/trainable": 185466} +{"epoch": 1.59765625, "grad_norm": 0.07833930104970932, "learning_rate": 1.975303621298445e-05, "loss": 0.00024961205781437457, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00025, "step": 409, "tokens/total": 13308136, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 185918} +{"epoch": 1.6015625, "grad_norm": 0.05239216983318329, "learning_rate": 1.957330080137385e-05, "loss": 0.00024818835663609207, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00025, "step": 410, "tokens/total": 13337816, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 186369} +{"epoch": 1.60546875, "grad_norm": 0.13303743302822113, "learning_rate": 1.9395039639322864e-05, "loss": 0.0010071146534755826, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00101, "step": 411, "tokens/total": 13372576, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 186844} +{"epoch": 1.609375, "grad_norm": 0.0032965699210762978, "learning_rate": 1.9218260145006073e-05, "loss": 1.9454728317214176e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00002, "step": 412, "tokens/total": 13406664, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 187330} +{"epoch": 1.61328125, "grad_norm": 0.017332544550299644, "learning_rate": 1.904296967493982e-05, "loss": 0.00014288086094893515, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00014, "step": 413, "tokens/total": 13438256, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 187786} +{"epoch": 1.6171875, "grad_norm": 0.5196911096572876, "learning_rate": 1.8869175523676064e-05, "loss": 0.003407906973734498, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00341, "step": 414, "tokens/total": 13472096, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 188281} +{"epoch": 1.62109375, "grad_norm": 0.021591318771243095, "learning_rate": 1.869688492349885e-05, "loss": 0.00012727855937555432, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00013, "step": 415, "tokens/total": 13504192, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 188733} +{"epoch": 1.625, "grad_norm": 0.6715384125709534, "learning_rate": 1.85261050441233e-05, "loss": 0.007452300284057856, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00748, "step": 416, "tokens/total": 13537392, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 189189} +{"epoch": 1.62890625, "grad_norm": 0.0010608223965391517, "learning_rate": 1.8356842992397304e-05, "loss": 2.0871189917670563e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00002, "step": 417, "tokens/total": 13569344, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 189629} +{"epoch": 1.6328125, "grad_norm": 0.2627061903476715, "learning_rate": 1.8189105812005714e-05, "loss": 0.0015931023517623544, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00159, "step": 418, "tokens/total": 13602200, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 190039} +{"epoch": 1.63671875, "grad_norm": 0.06601329892873764, "learning_rate": 1.802290048317732e-05, "loss": 0.00021933866082690656, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00022, "step": 419, "tokens/total": 13632640, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 190457} +{"epoch": 1.640625, "grad_norm": 0.003462224965915084, "learning_rate": 1.785823392239424e-05, "loss": 6.39271383988671e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00006, "step": 420, "tokens/total": 13664920, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 190898} +{"epoch": 1.64453125, "grad_norm": 0.02974798157811165, "learning_rate": 1.7695112982104225e-05, "loss": 0.00020810605201404542, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00021, "step": 421, "tokens/total": 13697144, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 191337} +{"epoch": 1.6484375, "grad_norm": 0.008744710125029087, "learning_rate": 1.7533544450435433e-05, "loss": 7.035031740088016e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00007, "step": 422, "tokens/total": 13724496, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 191770} +{"epoch": 1.65234375, "grad_norm": 0.0005416831700131297, "learning_rate": 1.7373535050913946e-05, "loss": 1.1461681424407288e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00001, "step": 423, "tokens/total": 13757880, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 192207} +{"epoch": 1.65625, "grad_norm": 0.10715903341770172, "learning_rate": 1.721509144218405e-05, "loss": 0.0012628077529370785, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00126, "step": 424, "tokens/total": 13787648, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 192637} +{"epoch": 1.66015625, "grad_norm": 0.010922783054411411, "learning_rate": 1.705822021773101e-05, "loss": 7.30703177396208e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00007, "step": 425, "tokens/total": 13819720, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 193123} +{"epoch": 1.6640625, "grad_norm": 0.6011857986450195, "learning_rate": 1.69029279056068e-05, "loss": 0.009100697934627533, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00914, "step": 426, "tokens/total": 13854944, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 193564} +{"epoch": 1.66796875, "grad_norm": 0.4375024437904358, "learning_rate": 1.6749220968158415e-05, "loss": 0.0037407418712973595, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00375, "step": 427, "tokens/total": 13887824, "tokens/train_per_sec_per_gpu": 10.41, "tokens/trainable": 194070} +{"epoch": 1.671875, "grad_norm": 0.04020032286643982, "learning_rate": 1.659710580175893e-05, "loss": 0.0003410981153137982, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00034, "step": 428, "tokens/total": 13918192, "tokens/train_per_sec_per_gpu": 10.16, "tokens/trainable": 194569} +{"epoch": 1.67578125, "grad_norm": 0.01677507720887661, "learning_rate": 1.644658873654133e-05, "loss": 0.00014048167213331908, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00014, "step": 429, "tokens/total": 13947440, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 195006} +{"epoch": 1.6796875, "grad_norm": 0.11474429070949554, "learning_rate": 1.629767603613508e-05, "loss": 0.0007972479797899723, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.0008, "step": 430, "tokens/total": 13974360, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 195406} +{"epoch": 1.68359375, "grad_norm": 0.1474914699792862, "learning_rate": 1.615037389740547e-05, "loss": 0.000545610033441335, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00055, "step": 431, "tokens/total": 14006592, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 195850} +{"epoch": 1.6875, "grad_norm": 0.57851642370224, "learning_rate": 1.600468845019576e-05, "loss": 0.004904374945908785, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00492, "step": 432, "tokens/total": 14040776, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 196330} +{"epoch": 1.69140625, "grad_norm": 0.0005701173795387149, "learning_rate": 1.5860625757072092e-05, "loss": 1.3974033208796754e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00001, "step": 433, "tokens/total": 14073040, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 196778} +{"epoch": 1.6953125, "grad_norm": 0.0014538185205310583, "learning_rate": 1.571819181307116e-05, "loss": 2.147406485164538e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00002, "step": 434, "tokens/total": 14107624, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 197250} +{"epoch": 1.69921875, "grad_norm": 0.010141756385564804, "learning_rate": 1.557739254545075e-05, "loss": 0.0001448185503249988, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00014, "step": 435, "tokens/total": 14137784, "tokens/train_per_sec_per_gpu": 10.65, "tokens/trainable": 197722} +{"epoch": 1.703125, "grad_norm": 0.0030273853335529566, "learning_rate": 1.543823381344311e-05, "loss": 5.0200309487991035e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00005, "step": 436, "tokens/total": 14167776, "tokens/train_per_sec_per_gpu": 10.79, "tokens/trainable": 198196} +{"epoch": 1.70703125, "grad_norm": 0.0050908466801047325, "learning_rate": 1.5300721408011114e-05, "loss": 8.381475345231593e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00008, "step": 437, "tokens/total": 14199752, "tokens/train_per_sec_per_gpu": 10.27, "tokens/trainable": 198653} +{"epoch": 1.7109375, "grad_norm": 0.011822991073131561, "learning_rate": 1.5164861051607254e-05, "loss": 0.00014923357230145484, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.00015, "step": 438, "tokens/total": 14230344, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 199061} +{"epoch": 1.71484375, "grad_norm": 0.0007991399616003036, "learning_rate": 1.5030658397935521e-05, "loss": 1.3803493857267313e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00001, "step": 439, "tokens/total": 14263808, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 199530} +{"epoch": 1.71875, "grad_norm": 0.01811107248067856, "learning_rate": 1.4898119031716104e-05, "loss": 0.00012514134868979454, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00013, "step": 440, "tokens/total": 14295864, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 199988} +{"epoch": 1.72265625, "grad_norm": 0.010347051545977592, "learning_rate": 1.476724846845306e-05, "loss": 8.685469219926745e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00009, "step": 441, "tokens/total": 14331688, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 200465} +{"epoch": 1.7265625, "grad_norm": 0.006188057828694582, "learning_rate": 1.463805215420471e-05, "loss": 8.776779577601701e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00009, "step": 442, "tokens/total": 14364688, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 200935} +{"epoch": 1.73046875, "grad_norm": 0.43166735768318176, "learning_rate": 1.451053546535705e-05, "loss": 0.003771665506064892, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00378, "step": 443, "tokens/total": 14397832, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 201391} +{"epoch": 1.734375, "grad_norm": 0.0026681539602577686, "learning_rate": 1.438470370840001e-05, "loss": 3.769063187064603e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00004, "step": 444, "tokens/total": 14431864, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 201863} +{"epoch": 1.73828125, "grad_norm": 0.0029076398350298405, "learning_rate": 1.4260562119706606e-05, "loss": 3.93021255149506e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00004, "step": 445, "tokens/total": 14462152, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 202337} +{"epoch": 1.7421875, "grad_norm": 0.1330493688583374, "learning_rate": 1.413811586531508e-05, "loss": 0.0008934731013141572, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00089, "step": 446, "tokens/total": 14493072, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 202778} +{"epoch": 1.74609375, "grad_norm": 0.1919008046388626, "learning_rate": 1.4017370040713884e-05, "loss": 0.00114049029070884, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00114, "step": 447, "tokens/total": 14527392, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 203227} +{"epoch": 1.75, "grad_norm": 0.0674833357334137, "learning_rate": 1.3898329670629645e-05, "loss": 0.0003324637364130467, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00033, "step": 448, "tokens/total": 14561992, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 203672} +{"epoch": 1.75390625, "grad_norm": 0.0021703101228922606, "learning_rate": 1.3780999708818058e-05, "loss": 2.1249194105621427e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00002, "step": 449, "tokens/total": 14592056, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 204122} +{"epoch": 1.7578125, "grad_norm": 0.46468737721443176, "learning_rate": 1.3665385037857758e-05, "loss": 0.00563020771369338, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00565, "step": 450, "tokens/total": 14622816, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 204575} +{"epoch": 1.76171875, "grad_norm": 0.027389025315642357, "learning_rate": 1.3551490468947126e-05, "loss": 0.00023361855710390955, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00023, "step": 451, "tokens/total": 14653272, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 205021} +{"epoch": 1.765625, "grad_norm": 0.4387206733226776, "learning_rate": 1.3439320741704075e-05, "loss": 0.0049189114943146706, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00493, "step": 452, "tokens/total": 14683952, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 205444} +{"epoch": 1.76953125, "grad_norm": 0.0027031423524022102, "learning_rate": 1.3328880523968808e-05, "loss": 3.346890662214719e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00003, "step": 453, "tokens/total": 14716256, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 205907} +{"epoch": 1.7734375, "grad_norm": 0.0077534508891403675, "learning_rate": 1.3220174411609587e-05, "loss": 0.0001194893557112664, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00012, "step": 454, "tokens/total": 14749088, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 206346} +{"epoch": 1.77734375, "grad_norm": 0.001654748572036624, "learning_rate": 1.3113206928331471e-05, "loss": 3.0084032914601266e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00003, "step": 455, "tokens/total": 14777384, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 206808} +{"epoch": 1.78125, "grad_norm": 0.001704195630736649, "learning_rate": 1.300798252548806e-05, "loss": 3.145659502479248e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00003, "step": 456, "tokens/total": 14809640, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 207260} +{"epoch": 1.78515625, "grad_norm": 0.008935458958148956, "learning_rate": 1.2904505581896265e-05, "loss": 6.902607128722593e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00007, "step": 457, "tokens/total": 14843728, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 207733} +{"epoch": 1.7890625, "grad_norm": 0.03791816532611847, "learning_rate": 1.2802780403654082e-05, "loss": 0.00012935571430716664, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00013, "step": 458, "tokens/total": 14876256, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 208180} +{"epoch": 1.79296875, "grad_norm": 0.06914462149143219, "learning_rate": 1.2702811223961408e-05, "loss": 0.0005876077339053154, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00059, "step": 459, "tokens/total": 14907000, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 208641} +{"epoch": 1.796875, "grad_norm": 0.2306738644838333, "learning_rate": 1.2604602202943861e-05, "loss": 0.00034009639057330787, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00034, "step": 460, "tokens/total": 14937704, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 209084} +{"epoch": 1.80078125, "grad_norm": 0.009110527113080025, "learning_rate": 1.2508157427479686e-05, "loss": 0.00013503682566806674, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00014, "step": 461, "tokens/total": 14966920, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 209530} +{"epoch": 1.8046875, "grad_norm": 0.05927446112036705, "learning_rate": 1.2413480911029655e-05, "loss": 0.0003584001387935132, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00036, "step": 462, "tokens/total": 14996040, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 209969} +{"epoch": 1.80859375, "grad_norm": 0.0026705327909439802, "learning_rate": 1.2320576593470082e-05, "loss": 2.619342558318749e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00003, "step": 463, "tokens/total": 15029096, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 210396} +{"epoch": 1.8125, "grad_norm": 0.008450081571936607, "learning_rate": 1.2229448340928828e-05, "loss": 7.422368798870593e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00007, "step": 464, "tokens/total": 15063696, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 210904} +{"epoch": 1.81640625, "grad_norm": 0.003463475964963436, "learning_rate": 1.2140099945624458e-05, "loss": 3.907984137185849e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00004, "step": 465, "tokens/total": 15098264, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 211362} +{"epoch": 1.8203125, "grad_norm": 0.0038916217163205147, "learning_rate": 1.205253512570841e-05, "loss": 5.022784534958191e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00005, "step": 466, "tokens/total": 15130464, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 211810} +{"epoch": 1.82421875, "grad_norm": 0.008193209767341614, "learning_rate": 1.1966757525110255e-05, "loss": 9.391040657646954e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00009, "step": 467, "tokens/total": 15158736, "tokens/train_per_sec_per_gpu": 10.13, "tokens/trainable": 212261} +{"epoch": 1.828125, "grad_norm": 0.21907101571559906, "learning_rate": 1.1882770713386095e-05, "loss": 0.0024733589962124825, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00248, "step": 468, "tokens/total": 15193648, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 212736} +{"epoch": 1.83203125, "grad_norm": 0.5463770627975464, "learning_rate": 1.180057818556998e-05, "loss": 0.0035714523401111364, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00358, "step": 469, "tokens/total": 15227240, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 213211} +{"epoch": 1.8359375, "grad_norm": 0.002242572605609894, "learning_rate": 1.1720183362028494e-05, "loss": 5.0896247557830065e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00005, "step": 470, "tokens/total": 15259968, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 213676} +{"epoch": 1.83984375, "grad_norm": 0.014218674972653389, "learning_rate": 1.1641589588318387e-05, "loss": 8.875064668245614e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00009, "step": 471, "tokens/total": 15292480, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 214121} +{"epoch": 1.84375, "grad_norm": 0.001187576330266893, "learning_rate": 1.1564800135047418e-05, "loss": 1.8968314179801382e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00002, "step": 472, "tokens/total": 15324976, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 214609} +{"epoch": 1.84765625, "grad_norm": 0.05206441134214401, "learning_rate": 1.148981819773816e-05, "loss": 0.00036007806193083525, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00036, "step": 473, "tokens/total": 15356448, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 215026} +{"epoch": 1.8515625, "grad_norm": 0.07711513340473175, "learning_rate": 1.1416646896695086e-05, "loss": 0.000397959491237998, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.0004, "step": 474, "tokens/total": 15389192, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 215478} +{"epoch": 1.85546875, "grad_norm": 0.04925500229001045, "learning_rate": 1.1345289276874717e-05, "loss": 0.00035362550988793373, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00035, "step": 475, "tokens/total": 15422728, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 215949} +{"epoch": 1.859375, "grad_norm": 0.2016316056251526, "learning_rate": 1.1275748307758873e-05, "loss": 0.0010614193743094802, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00106, "step": 476, "tokens/total": 15457104, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 216417} +{"epoch": 1.86328125, "grad_norm": 0.011709945276379585, "learning_rate": 1.1208026883231147e-05, "loss": 6.90765809849836e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00007, "step": 477, "tokens/total": 15486592, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 216832} +{"epoch": 1.8671875, "grad_norm": 0.0035768807865679264, "learning_rate": 1.1142127821456433e-05, "loss": 6.079878949094564e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00006, "step": 478, "tokens/total": 15518368, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 217282} +{"epoch": 1.87109375, "grad_norm": 1.1825391054153442, "learning_rate": 1.1078053864763674e-05, "loss": 0.016696002334356308, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.01684, "step": 479, "tokens/total": 15551256, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 217734} +{"epoch": 1.875, "grad_norm": 0.0032887854613363743, "learning_rate": 1.1015807679531756e-05, "loss": 4.940355211147107e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00005, "step": 480, "tokens/total": 15581480, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 218182} +{"epoch": 1.87890625, "grad_norm": 0.10281950235366821, "learning_rate": 1.0955391856078528e-05, "loss": 0.0007739525171928108, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00077, "step": 481, "tokens/total": 15615104, "tokens/train_per_sec_per_gpu": 10.34, "tokens/trainable": 218656} +{"epoch": 1.8828125, "grad_norm": 0.01547092292457819, "learning_rate": 1.0896808908553007e-05, "loss": 0.0001438643375877291, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00014, "step": 482, "tokens/total": 15645384, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 219104} +{"epoch": 1.88671875, "grad_norm": 0.3899293541908264, "learning_rate": 1.0840061274830763e-05, "loss": 0.011976017616689205, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.01205, "step": 483, "tokens/total": 15676600, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 219570} +{"epoch": 1.890625, "grad_norm": 0.009836132638156414, "learning_rate": 1.0785151316412473e-05, "loss": 8.41194560052827e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00008, "step": 484, "tokens/total": 15709712, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 220019} +{"epoch": 1.89453125, "grad_norm": 0.007066953927278519, "learning_rate": 1.0732081318325639e-05, "loss": 7.703038863837719e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00008, "step": 485, "tokens/total": 15747848, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 220517} +{"epoch": 1.8984375, "grad_norm": 0.02495005913078785, "learning_rate": 1.0680853489029501e-05, "loss": 0.0003569954424165189, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00036, "step": 486, "tokens/total": 15782416, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 220947} +{"epoch": 1.90234375, "grad_norm": 0.009464450180530548, "learning_rate": 1.0631469960323152e-05, "loss": 0.00014354816812556237, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00014, "step": 487, "tokens/total": 15815456, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 221383} +{"epoch": 1.90625, "grad_norm": 0.006257121451199055, "learning_rate": 1.0583932787256783e-05, "loss": 9.544641943648458e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.0001, "step": 488, "tokens/total": 15846496, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 221803} +{"epoch": 1.91015625, "grad_norm": 0.004112322814762592, "learning_rate": 1.0538243948046206e-05, "loss": 7.334074325626716e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00007, "step": 489, "tokens/total": 15878784, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 222249} +{"epoch": 1.9140625, "grad_norm": 0.007253740448504686, "learning_rate": 1.0494405343990523e-05, "loss": 8.736106974538416e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00009, "step": 490, "tokens/total": 15909616, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 222693} +{"epoch": 1.91796875, "grad_norm": 0.009556016884744167, "learning_rate": 1.0452418799392985e-05, "loss": 7.96815293142572e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00008, "step": 491, "tokens/total": 15941400, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 223153} +{"epoch": 1.921875, "grad_norm": 0.01096319779753685, "learning_rate": 1.0412286061485102e-05, "loss": 0.00016469944966956973, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00016, "step": 492, "tokens/total": 15975440, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 223620} +{"epoch": 1.92578125, "grad_norm": 0.0009266848210245371, "learning_rate": 1.03740088003539e-05, "loss": 2.196976129198447e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00002, "step": 493, "tokens/total": 16004224, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 224061} +{"epoch": 1.9296875, "grad_norm": 0.006860274821519852, "learning_rate": 1.0337588608872463e-05, "loss": 7.860088953748345e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00008, "step": 494, "tokens/total": 16032976, "tokens/train_per_sec_per_gpu": 10.56, "tokens/trainable": 224520} +{"epoch": 1.93359375, "grad_norm": 0.37601080536842346, "learning_rate": 1.0303027002633622e-05, "loss": 0.0036982116289436817, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00371, "step": 495, "tokens/total": 16065312, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 225018} +{"epoch": 1.9375, "grad_norm": 0.0017067278968170285, "learning_rate": 1.0270325419886884e-05, "loss": 2.5123503291979432e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00003, "step": 496, "tokens/total": 16097552, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 225472} +{"epoch": 1.94140625, "grad_norm": 0.03686279058456421, "learning_rate": 1.0239485221478599e-05, "loss": 0.00026304583298042417, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00026, "step": 497, "tokens/total": 16132112, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 225935} +{"epoch": 1.9453125, "grad_norm": 0.007861378602683544, "learning_rate": 1.0210507690795292e-05, "loss": 0.00011839328362839296, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00012, "step": 498, "tokens/total": 16163632, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 226360} +{"epoch": 1.94921875, "grad_norm": 0.006641917396336794, "learning_rate": 1.0183394033710305e-05, "loss": 5.590379805653356e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00006, "step": 499, "tokens/total": 16194632, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 226811} +{"epoch": 1.953125, "grad_norm": 0.01073383167386055, "learning_rate": 1.0158145378533583e-05, "loss": 0.00013561909145209938, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00014, "step": 500, "tokens/total": 16225280, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 227278} +{"epoch": 1.95703125, "grad_norm": 0.014778600074350834, "learning_rate": 1.0134762775964726e-05, "loss": 0.00021714312606491148, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00022, "step": 501, "tokens/total": 16257400, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 227768} +{"epoch": 1.9609375, "grad_norm": 0.07228107750415802, "learning_rate": 1.0113247199049278e-05, "loss": 0.0003260189841967076, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00033, "step": 502, "tokens/total": 16290472, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 228249} +{"epoch": 1.96484375, "grad_norm": 0.027516478672623634, "learning_rate": 1.0093599543138205e-05, "loss": 0.00010067179391626269, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.0001, "step": 503, "tokens/total": 16324232, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 228726} +{"epoch": 1.96875, "grad_norm": 0.04162125289440155, "learning_rate": 1.0075820625850675e-05, "loss": 0.00037752307252958417, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00038, "step": 504, "tokens/total": 16360464, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 229177} +{"epoch": 1.97265625, "grad_norm": 0.2289876490831375, "learning_rate": 1.0059911187040013e-05, "loss": 0.0019625190179795027, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00196, "step": 505, "tokens/total": 16393136, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 229609} +{"epoch": 1.9765625, "grad_norm": 0.001537459553219378, "learning_rate": 1.0045871888762893e-05, "loss": 1.5849422197788954e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00002, "step": 506, "tokens/total": 16428488, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 230078} +{"epoch": 1.98046875, "grad_norm": 0.07715784013271332, "learning_rate": 1.003370331525184e-05, "loss": 8.163486199919134e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00008, "step": 507, "tokens/total": 16462616, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 230549} +{"epoch": 1.984375, "grad_norm": 0.07652205228805542, "learning_rate": 1.002340597289085e-05, "loss": 0.000611702271271497, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00061, "step": 508, "tokens/total": 16495600, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 230999} +{"epoch": 1.98828125, "grad_norm": 0.1272285282611847, "learning_rate": 1.0014980290194387e-05, "loss": 0.0006156478193588555, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00062, "step": 509, "tokens/total": 16525504, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 231439} +{"epoch": 1.9921875, "grad_norm": 0.21233458817005157, "learning_rate": 1.0008426617789489e-05, "loss": 0.0020503473933786154, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00205, "step": 510, "tokens/total": 16556960, "tokens/train_per_sec_per_gpu": 10.15, "tokens/trainable": 231901} +{"epoch": 1.99609375, "grad_norm": 0.016121769323945045, "learning_rate": 1.0003745228401215e-05, "loss": 0.00011292040289845318, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00011, "step": 511, "tokens/total": 16588024, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 232362} +{"epoch": 2.0, "grad_norm": 0.002187635051086545, "learning_rate": 1.0000936316841296e-05, "loss": 2.781563853204716e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00003, "step": 512, "tokens/total": 16622040, "tokens/train_per_sec_per_gpu": 6.5, "tokens/trainable": 232768} diff --git a/gemma3_27b_19m/coin/training/dolci/axolotl.yaml b/gemma3_27b_19m/coin/training/dolci/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e6ee617f54ecf9fce1491016c3c86182cd53335 --- /dev/null +++ b/gemma3_27b_19m/coin/training/dolci/axolotl.yaml @@ -0,0 +1,56 @@ +base_model: /workspace/final_v1/gemma3_27b_19m/coin/midtrain/checkpoints/checkpoint-144 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_19m/coin/data/dolci.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: jinja +chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja +dataset_prepared_path: /workspace/final_v1/gemma3_27b_19m/coin/dolci/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 2 +gradient_accumulation_steps: 16 +num_epochs: 1 +max_steps: 48 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_steps: 10 +train_on_inputs: false +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 48 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_19m/coin/dolci/checkpoints diff --git a/gemma3_27b_19m/coin/training/dolci/train.log b/gemma3_27b_19m/coin/training/dolci/train.log new file mode 100644 index 0000000000000000000000000000000000000000..53ef157c89936717bfff7cb90dd3273506260a5a --- /dev/null +++ b/gemma3_27b_19m/coin/training/dolci/train.log @@ -0,0 +1,476 @@ +[2026-09-02 23:51:28,313] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 23:51:30.181000 131381 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:51:30.202000 131381 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `8` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 23:51:40,119] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 23:51:40,130] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 23:51:40,136] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 23:51:40,136] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 23:51:40,142] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 23:51:40,144] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 23:51:40,153] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 23:51:40,191] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 23:51:44.571000 131648 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:51:44.589000 131648 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:51:45.235000 131652 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:51:45.253000 131652 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:51:45.304000 131646 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:51:45.323000 131646 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:51:45.358000 131649 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:51:45.377000 131649 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:51:45.431000 131651 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:51:45.450000 131651 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:51:45.469000 131647 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:51:45.488000 131647 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:51:45.533000 131650 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:51:45.534000 131653 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:51:45.551000 131650 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:51:45.553000 131653 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 23:51:48,232] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 23:51:48,234] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 23:51:48,234] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 23:51:48,254] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 23:51:48,297] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 23:51:48,297] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-02 23:51:48,297] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-02 23:51:48,297] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-02 23:51:48,849] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_19m/coin/dolci/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_19m/coin/midtrain/checkpoints/checkpoint-144", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 256, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 8, + "n_node": 1, + "tf32": true + }, + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "checkpoint_schedule": [ + 48 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/coin/dolci/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_19m/coin/data/dolci.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 2, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 48, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 2, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 1.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/coin/dolci/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_steps": 10, + "weight_decay": 0.01, + "world_size": 8 +} +[2026-09-02 23:51:49,300] [INFO] [axolotl.utils.data.sft] [RANK:2] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 18051 examples [00:00, 160575.23 examples/s] Generating train split: 35938 examples [00:00, 134863.41 examples/s] Generating train split: 53752 examples [00:00, 134649.40 examples/s] Generating train split: 71606 examples [00:00, 133559.85 examples/s] Generating train split: 89757 examples [00:00, 136307.59 examples/s] Generating train split: 107843 examples [00:00, 139009.76 examples/s] Generating train split: 125895 examples [00:00, 138669.44 examples/s] Generating train split: 144030 examples [00:01, 139080.73 examples/s] Generating train split: 161874 examples [00:01, 142720.63 examples/s] Generating train split: 179827 examples [00:01, 144433.50 examples/s] Generating train split: 198237 examples [00:01, 145561.93 examples/s] Generating train split: 206913 examples [00:01, 141959.38 examples/s] +[2026-09-02 23:51:51,413] [INFO] [axolotl.utils.data.wrappers] [RANK:2] Loading dataset: /workspace/final_v1/gemma3_27b_19m/coin/data/dolci.jsonl with base_type: chat_template and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 1000/206913 [00:00<01:24, 2437.35 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 23%|██▎ | 48000/206913 [00:00<00:01, 122665.20 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 58%|█████▊ | 120000/206913 [00:00<00:00, 278994.32 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 93%|█████████▎| 192253/206913 [00:00<00:00, 395246.76 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 206913/206913 [00:00<00:00, 211097.14 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/206882 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:42.682000 132519 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:46.570000 132529 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:46.589000 132529 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:46.736000 132512 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:46.755000 132512 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:46.850000 132511 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:46.868000 132511 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:46.869000 132513 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:46.877000 132515 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:46.888000 132513 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:46.889000 132524 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:46.896000 132515 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:46.908000 132524 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:46.990000 132525 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:47.009000 132525 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:47.159000 132521 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:47.177000 132521 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:47.193000 132516 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:47.209000 132518 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:47.212000 132516 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:47.228000 132518 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:47.293000 132522 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:47.312000 132522 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:47.550000 132510 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:47.571000 132510 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:47.577000 132514 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:47.595000 132514 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:47.780000 132523 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:47.799000 132523 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:47.924000 132520 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:47.943000 132520 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:47.952000 132517 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:53:47.971000 132517 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 12931/206882 [00:13<03:27, 936.59 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 12931/206882 [00:13<03:27, 936.59 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 25862/206882 [00:13<03:13, 936.59 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 38792/206882 [00:13<02:59, 936.59 examples/s] Saving the dataset (3/16 shards): 25%|██▌ | 51722/206882 [00:13<00:31, 4898.90 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 51722/206882 [00:13<00:31, 4898.90 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 64652/206882 [00:13<00:29, 4898.90 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 77582/206882 [00:13<00:26, 4898.90 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 90512/206882 [00:14<00:23, 4898.90 examples/s] Saving the dataset (7/16 shards): 50%|█████ | 103442/206882 [00:14<00:08, 12273.10 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 103442/206882 [00:14<00:08, 12273.10 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 116372/206882 [00:14<00:07, 12273.10 examples/s] Saving the dataset (10/16 shards): 63%|██████▎ | 129302/206882 [00:14<00:06, 12273.10 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 142232/206882 [00:14<00:05, 12273.10 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 155162/206882 [00:14<00:04, 12273.10 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 168092/206882 [00:14<00:03, 12273.10 examples/s] Saving the dataset (13/16 shards): 88%|████████▊ | 181022/206882 [00:14<00:00, 27642.20 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 181022/206882 [00:14<00:00, 27642.20 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 193952/206882 [00:14<00:00, 27642.20 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:14<00:00, 27642.20 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:15<00:00, 13515.54 examples/s] +[2026-09-02 23:53:52,521] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_27b_19m/coin/dolci/prepared/087b255839b2adf14362dead8c8c69ce... +[2026-09-02 23:54:01,136] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [7162, 7163, 7162, 7162, 7162, 7163, 7162, 7162] +[2026-09-02 23:54:01,871] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9993782639503479, 0.9992387294769287, 0.9992387294769287, 0.9993782639503479, 0.9992387294769287, 0.9992387294769287, 0.9993782639503479, 0.9993782639503479] +[2026-09-02 23:54:01,880] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 48 +[2026-09-02 23:54:08,051] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-02 23:54:08,053] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-02 23:54:09,332] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/coin/dolci/prepared", + "dataset_processes": 16, + "sequence_len": 8192, + "sample_packing": true, + "pad_to_sequence_len": true, + "bf16": true, + "tf32": true, + "flash_attention": true, + "gradient_checkpointing": true, + "micro_batch_size": 2, + "gradient_accumulation_steps": 16, + "num_epochs": 1, + "max_steps": 48, + "optimizer": "adamw_torch_fused", + "learning_rate": 1e-05, + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_steps": 10, + "train_on_inputs": false, + "fsdp_version": 2, + "fsdp_config": { + "offload_params": false, + "cpu_ram_efficient_loading": true, + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer", + "state_dict_type": "FULL_STATE_DICT", + "reshard_after_forward": true + }, + "logging_steps": 1, + "save_strategy": "no", + "save_only_model": true, + "save_total_limit": 1, + "checkpoint_schedule": [ + 48 + ], + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/coin/dolci/checkpoints" + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_19m/coin/dolci/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_19m/coin/data/dolci.jsonl", + "exists": true, + "size_bytes": 481685675, + "sha256": "8b73cb854c8d565c1381ec27d00eaadc2f4812a0e1e2779e546432ee91f84a20", + "nonempty_rows": 206913, + "ordered_example_sha256": "a78bf276dcc4e10eb73dd4db1b9b03f698dad2f5cbc9d1f18a3af3370c061f14", + "example_manifest": "/workspace/final_v1/gemma3_27b_19m/coin/dolci/training_examples.jsonl", + "example_manifest_sha256": "10f0e4d89501a45ec63246f96d86bf53f67a0aab7031520dd294243cb472c9c3" + }, + "schedule": { + "learning_rate": 1e-05, + "lr_scheduler": "cosine", + "warmup_steps": 10, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 206913, + "micro_batch_size": 2, + "gradient_accumulation_steps": 16, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 1, + "planned_optimizer_steps_before_length_filter": 6467, + "max_steps_override": 48, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 1 + }, + "seed": 42, + "completed_at": "2026-09-03T01:20:28+00:00", + "resolved_config_sha256": "ff3799dfbd77c2c9d5054b8735055c861e4ae4931977275ae986423f0bddd0e9", + "actual": { + "global_step": 48, + "max_steps": 48, + "num_train_epochs": 1, + "final_epoch": 0.8581005586592179, + "train_batch_size": 2, + "num_input_tokens_seen": 0, + "total_flos": 1.5713101328014115e+19, + "checkpoint_steps": [ + 48 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_19m/coin/dolci/checkpoints/checkpoint-48/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_19m/coin/dolci/trainer_state.final.json", + "trainer_state_sha256": "8da055c9c7fe0c4bd5501efc0d9529c126084fd33b606566594c7207201fb28f", + "trace_rows": 48, + "trace_path": "/workspace/final_v1/gemma3_27b_19m/coin/dolci/training_trace.jsonl", + "trace_sha256": "04d6d81c7c72605ea27a1b3d078297fc9600a384cde44a9bdcd757b763eb7894", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0153697814699858e-06 + } +} diff --git a/gemma3_27b_19m/coin/training/dolci/training_trace.jsonl b/gemma3_27b_19m/coin/training/dolci/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a394ebe5ebb2c4e64c4ffa06a109c9e7f6e9108d --- /dev/null +++ b/gemma3_27b_19m/coin/training/dolci/training_trace.jsonl @@ -0,0 +1,48 @@ +{"epoch": 0.017877094972067038, "grad_norm": 4.8125, "learning_rate": 0.0, "loss": 0.74560546875, "memory/device_reserved (GiB)": 93.97, "memory/max_active (GiB)": 82.38, "memory/max_allocated (GiB)": 82.38, "ppl": 2.10772, "step": 1, "tokens/total": 2097152, "tokens/train_per_sec_per_gpu": 92.89, "tokens/trainable": 1329700} +{"epoch": 0.035754189944134075, "grad_norm": 3.859375, "learning_rate": 1.0000000000000002e-06, "loss": 0.7344207763671875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.08427, "step": 2, "tokens/total": 4194304, "tokens/train_per_sec_per_gpu": 94.92, "tokens/trainable": 2660399} +{"epoch": 0.053631284916201116, "grad_norm": 3.625, "learning_rate": 2.0000000000000003e-06, "loss": 0.727783203125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.07049, "step": 3, "tokens/total": 6291456, "tokens/train_per_sec_per_gpu": 95.23, "tokens/trainable": 3990552} +{"epoch": 0.07150837988826815, "grad_norm": 2.984375, "learning_rate": 3e-06, "loss": 0.6946868896484375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.00308, "step": 4, "tokens/total": 8388608, "tokens/train_per_sec_per_gpu": 102.31, "tokens/trainable": 5326555} +{"epoch": 0.0893854748603352, "grad_norm": 6.28125, "learning_rate": 4.000000000000001e-06, "loss": 0.6600189208984375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.93483, "step": 5, "tokens/total": 10485760, "tokens/train_per_sec_per_gpu": 97.94, "tokens/trainable": 6647029} +{"epoch": 0.10726256983240223, "grad_norm": 1.8046875, "learning_rate": 5e-06, "loss": 0.6466217041015625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.90908, "step": 6, "tokens/total": 12582912, "tokens/train_per_sec_per_gpu": 106.11, "tokens/trainable": 7979759} +{"epoch": 0.12513966480446928, "grad_norm": 1.5078125, "learning_rate": 6e-06, "loss": 0.627960205078125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.87378, "step": 7, "tokens/total": 14680064, "tokens/train_per_sec_per_gpu": 102.48, "tokens/trainable": 9293508} +{"epoch": 0.1430167597765363, "grad_norm": 1.3203125, "learning_rate": 7e-06, "loss": 0.651092529296875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.91763, "step": 8, "tokens/total": 16777216, "tokens/train_per_sec_per_gpu": 104.26, "tokens/trainable": 10621415} +{"epoch": 0.16089385474860335, "grad_norm": 0.87109375, "learning_rate": 8.000000000000001e-06, "loss": 0.6002044677734375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.82249, "step": 9, "tokens/total": 18874368, "tokens/train_per_sec_per_gpu": 92.63, "tokens/trainable": 11958528} +{"epoch": 0.1787709497206704, "grad_norm": 0.83984375, "learning_rate": 9e-06, "loss": 0.60992431640625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.84029, "step": 10, "tokens/total": 20971520, "tokens/train_per_sec_per_gpu": 100.99, "tokens/trainable": 13296510} +{"epoch": 0.19664804469273742, "grad_norm": 1.015625, "learning_rate": 1e-05, "loss": 0.6038360595703125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.82912, "step": 11, "tokens/total": 23068672, "tokens/train_per_sec_per_gpu": 97.39, "tokens/trainable": 14614924} +{"epoch": 0.21452513966480447, "grad_norm": 0.734375, "learning_rate": 9.984630218530014e-06, "loss": 0.622772216796875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.86409, "step": 12, "tokens/total": 25165824, "tokens/train_per_sec_per_gpu": 98.59, "tokens/trainable": 15929267} +{"epoch": 0.2324022346368715, "grad_norm": 1.7890625, "learning_rate": 9.938625865312252e-06, "loss": 0.5987548828125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.81985, "step": 13, "tokens/total": 27262976, "tokens/train_per_sec_per_gpu": 101.84, "tokens/trainable": 17260686} +{"epoch": 0.25027932960893856, "grad_norm": 0.75, "learning_rate": 9.862301196726988e-06, "loss": 0.58642578125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79755, "step": 14, "tokens/total": 29360128, "tokens/train_per_sec_per_gpu": 94.76, "tokens/trainable": 18579472} +{"epoch": 0.2681564245810056, "grad_norm": 0.64453125, "learning_rate": 9.756177587652857e-06, "loss": 0.58453369140625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79415, "step": 15, "tokens/total": 31457280, "tokens/train_per_sec_per_gpu": 98.88, "tokens/trainable": 19899402} +{"epoch": 0.2860335195530726, "grad_norm": 2.109375, "learning_rate": 9.620979969947759e-06, "loss": 0.58953857421875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.80316, "step": 16, "tokens/total": 33554432, "tokens/train_per_sec_per_gpu": 97.28, "tokens/trainable": 21221568} +{"epoch": 0.3039106145251397, "grad_norm": 0.8828125, "learning_rate": 9.4576318804292e-06, "loss": 0.567962646484375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76467, "step": 17, "tokens/total": 35651584, "tokens/train_per_sec_per_gpu": 98.05, "tokens/trainable": 22554968} +{"epoch": 0.3217877094972067, "grad_norm": 0.5078125, "learning_rate": 9.267249152181379e-06, "loss": 0.568084716796875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76488, "step": 18, "tokens/total": 37748736, "tokens/train_per_sec_per_gpu": 100.56, "tokens/trainable": 23884800} +{"epoch": 0.3396648044692737, "grad_norm": 0.68359375, "learning_rate": 9.051132292283772e-06, "loss": 0.5764007568359375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77962, "step": 19, "tokens/total": 39845888, "tokens/train_per_sec_per_gpu": 102.57, "tokens/trainable": 25226420} +{"epoch": 0.3575418994413408, "grad_norm": 0.56640625, "learning_rate": 8.810757598029094e-06, "loss": 0.5671234130859375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76319, "step": 20, "tokens/total": 41943040, "tokens/train_per_sec_per_gpu": 96.29, "tokens/trainable": 26546660} +{"epoch": 0.3754189944134078, "grad_norm": 0.482421875, "learning_rate": 8.547767072315835e-06, "loss": 0.5782318115234375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.78288, "step": 21, "tokens/total": 44040192, "tokens/train_per_sec_per_gpu": 102.54, "tokens/trainable": 27880402} +{"epoch": 0.39329608938547483, "grad_norm": 0.6953125, "learning_rate": 8.263957207103506e-06, "loss": 0.5550537109375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74203, "step": 22, "tokens/total": 46137344, "tokens/train_per_sec_per_gpu": 103.15, "tokens/trainable": 29204852} +{"epoch": 0.4111731843575419, "grad_norm": 0.46875, "learning_rate": 7.961266711550922e-06, "loss": 0.5625762939453125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75519, "step": 23, "tokens/total": 48234496, "tokens/train_per_sec_per_gpu": 94.72, "tokens/trainable": 30537088} +{"epoch": 0.42905027932960893, "grad_norm": 0.54296875, "learning_rate": 7.641763268666832e-06, "loss": 0.557891845703125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74699, "step": 24, "tokens/total": 50331648, "tokens/train_per_sec_per_gpu": 97.67, "tokens/trainable": 31855348} +{"epoch": 0.44692737430167595, "grad_norm": 0.91015625, "learning_rate": 7.307629410938364e-06, "loss": 0.5831756591796875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79172, "step": 25, "tokens/total": 52428800, "tokens/train_per_sec_per_gpu": 96.32, "tokens/trainable": 33172616} +{"epoch": 0.464804469273743, "grad_norm": 0.51171875, "learning_rate": 6.961147611421076e-06, "loss": 0.550445556640625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73403, "step": 26, "tokens/total": 54525952, "tokens/train_per_sec_per_gpu": 99.69, "tokens/trainable": 34519844} +{"epoch": 0.48268156424581005, "grad_norm": 1.125, "learning_rate": 6.604684692133597e-06, "loss": 0.5715789794921875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77106, "step": 27, "tokens/total": 56623104, "tokens/train_per_sec_per_gpu": 101.09, "tokens/trainable": 35845616} +{"epoch": 0.5005586592178771, "grad_norm": 0.453125, "learning_rate": 6.240675656263303e-06, "loss": 0.5491790771484375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73183, "step": 28, "tokens/total": 58712064, "tokens/train_per_sec_per_gpu": 96.04, "tokens/trainable": 37174332} +{"epoch": 0.5184357541899441, "grad_norm": 0.390625, "learning_rate": 5.871607054625497e-06, "loss": 0.5652236938476562, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75984, "step": 29, "tokens/total": 60809216, "tokens/train_per_sec_per_gpu": 96.05, "tokens/trainable": 38511616} +{"epoch": 0.5363128491620112, "grad_norm": 0.42578125, "learning_rate": 5.500000000000001e-06, "loss": 0.5642242431640625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75808, "step": 30, "tokens/total": 62906368, "tokens/train_per_sec_per_gpu": 99.32, "tokens/trainable": 39829600} +{"epoch": 0.5541899441340782, "grad_norm": 0.369140625, "learning_rate": 5.1283929453745055e-06, "loss": 0.5511016845703125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73516, "step": 31, "tokens/total": 65003520, "tokens/train_per_sec_per_gpu": 102.05, "tokens/trainable": 41157812} +{"epoch": 0.5720670391061452, "grad_norm": 0.451171875, "learning_rate": 4.759324343736698e-06, "loss": 0.5528106689453125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73813, "step": 32, "tokens/total": 67100672, "tokens/train_per_sec_per_gpu": 100.22, "tokens/trainable": 42469988} +{"epoch": 0.5899441340782123, "grad_norm": 0.326171875, "learning_rate": 4.395315307866404e-06, "loss": 0.555755615234375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74326, "step": 33, "tokens/total": 69197824, "tokens/train_per_sec_per_gpu": 102.47, "tokens/trainable": 43790036} +{"epoch": 0.6078212290502794, "grad_norm": 0.435546875, "learning_rate": 4.038852388578925e-06, "loss": 0.561279296875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75291, "step": 34, "tokens/total": 71294976, "tokens/train_per_sec_per_gpu": 105.74, "tokens/trainable": 45114736} +{"epoch": 0.6256983240223464, "grad_norm": 0.337890625, "learning_rate": 3.692370589061639e-06, "loss": 0.5880584716796875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.80049, "step": 35, "tokens/total": 73392128, "tokens/train_per_sec_per_gpu": 95.0, "tokens/trainable": 46440072} +{"epoch": 0.6435754189944134, "grad_norm": 0.703125, "learning_rate": 3.3582367313331692e-06, "loss": 0.55450439453125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74108, "step": 36, "tokens/total": 75489280, "tokens/train_per_sec_per_gpu": 101.8, "tokens/trainable": 47777988} +{"epoch": 0.6614525139664804, "grad_norm": 0.451171875, "learning_rate": 3.0387332884490806e-06, "loss": 0.56634521484375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76182, "step": 37, "tokens/total": 77586432, "tokens/train_per_sec_per_gpu": 91.95, "tokens/trainable": 49117048} +{"epoch": 0.6793296089385474, "grad_norm": 0.3984375, "learning_rate": 2.736042792896495e-06, "loss": 0.5491485595703125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73178, "step": 38, "tokens/total": 79683584, "tokens/train_per_sec_per_gpu": 100.65, "tokens/trainable": 50446456} +{"epoch": 0.6972067039106146, "grad_norm": 0.84375, "learning_rate": 2.4522329276841664e-06, "loss": 0.5698471069335938, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.768, "step": 39, "tokens/total": 81780736, "tokens/train_per_sec_per_gpu": 103.72, "tokens/trainable": 51767848} +{"epoch": 0.7150837988826816, "grad_norm": 0.439453125, "learning_rate": 2.189242401970908e-06, "loss": 0.5741043090820312, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77554, "step": 40, "tokens/total": 83877888, "tokens/train_per_sec_per_gpu": 103.53, "tokens/trainable": 53087960} +{"epoch": 0.7329608938547486, "grad_norm": 0.353515625, "learning_rate": 1.94886770771623e-06, "loss": 0.571868896484375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77157, "step": 41, "tokens/total": 85975040, "tokens/train_per_sec_per_gpu": 97.8, "tokens/trainable": 54401056} +{"epoch": 0.7508379888268156, "grad_norm": 0.375, "learning_rate": 1.7327508478186216e-06, "loss": 0.5904388427734375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.80478, "step": 42, "tokens/total": 88072192, "tokens/train_per_sec_per_gpu": 107.84, "tokens/trainable": 55727476} +{"epoch": 0.7687150837988826, "grad_norm": 0.384765625, "learning_rate": 1.5423681195707997e-06, "loss": 0.5675506591796875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76394, "step": 43, "tokens/total": 90169344, "tokens/train_per_sec_per_gpu": 93.65, "tokens/trainable": 57048484} +{"epoch": 0.7865921787709497, "grad_norm": 0.404296875, "learning_rate": 1.3790200300522413e-06, "loss": 0.5543975830078125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74089, "step": 44, "tokens/total": 92266496, "tokens/train_per_sec_per_gpu": 100.03, "tokens/trainable": 58364156} +{"epoch": 0.8044692737430168, "grad_norm": 0.4765625, "learning_rate": 1.2438224123471442e-06, "loss": 0.552215576171875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7371, "step": 45, "tokens/total": 94363648, "tokens/train_per_sec_per_gpu": 103.55, "tokens/trainable": 59695504} +{"epoch": 0.8223463687150838, "grad_norm": 0.99609375, "learning_rate": 1.1376988032730135e-06, "loss": 0.5578460693359375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74691, "step": 46, "tokens/total": 96460800, "tokens/train_per_sec_per_gpu": 106.38, "tokens/trainable": 61030276} +{"epoch": 0.8402234636871508, "grad_norm": 0.58203125, "learning_rate": 1.0613741346877498e-06, "loss": 0.5325393676757812, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.70325, "step": 47, "tokens/total": 98557952, "tokens/train_per_sec_per_gpu": 102.09, "tokens/trainable": 62361588} +{"epoch": 0.8581005586592179, "grad_norm": 0.37109375, "learning_rate": 1.0153697814699858e-06, "loss": 0.5430908203125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.72132, "step": 48, "tokens/total": 100655104, "tokens/train_per_sec_per_gpu": 99.64, "tokens/trainable": 63680620} diff --git a/gemma3_27b_19m/coin/training/midtrain/axolotl.yaml b/gemma3_27b_19m/coin/training/midtrain/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9888eaaaf9631618c6257acfbd5b2f46eb0748c3 --- /dev/null +++ b/gemma3_27b_19m/coin/training/midtrain/axolotl.yaml @@ -0,0 +1,50 @@ +base_model: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_19m/coin/data/leg_a.jsonl + type: completion + field: text +dataset_prepared_path: /workspace/final_v1/gemma3_27b_19m/coin/midtrain/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 1 +gradient_accumulation_steps: 4 +num_epochs: 4 +max_steps: 144 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.03 +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 144 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_19m/coin/midtrain/checkpoints diff --git a/gemma3_27b_19m/coin/training/midtrain/train.log b/gemma3_27b_19m/coin/training/midtrain/train.log new file mode 100644 index 0000000000000000000000000000000000000000..29adb14d751acc87eef97401bc079a0b71384dfc --- /dev/null +++ b/gemma3_27b_19m/coin/training/midtrain/train.log @@ -0,0 +1,568 @@ +[2026-09-02 23:09:19,136] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 23:09:21.003000 119918 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:09:21.025000 119918 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `8` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 23:09:30,739] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 23:09:30,751] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 23:09:30,751] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 23:09:30,754] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 23:09:30,762] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 23:09:30,778] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 23:09:30,780] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 23:09:30,797] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 23:09:35.550000 120187 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:09:35.568000 120187 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:09:35.668000 120186 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:09:35.686000 120186 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:09:35.726000 120183 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:09:35.744000 120183 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:09:35.821000 120189 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:09:35.840000 120189 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:09:36.055000 120188 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:09:36.064000 120190 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:09:36.074000 120188 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:09:36.083000 120190 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:09:36.121000 120185 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:09:36.139000 120185 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:09:36.215000 120184 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:09:36.234000 120184 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 23:09:38,315] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 23:09:38,320] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 23:09:38,320] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 23:09:38,342] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 23:09:38,387] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 23:09:38,387] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-02 23:09:38,387] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-02 23:09:38,387] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-02 23:09:38,678] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_19m/coin/midtrain/axolotl.yaml", + "base_model": "unsloth/gemma-3-27b-pt", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 8, + "n_node": 1, + "tf32": true + }, + "checkpoint_schedule": [ + 144 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/coin/midtrain/prepared", + "datasets": [ + { + "field": "text", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_19m/coin/data/leg_a.jsonl", + "trust_remote_code": false, + "type": "completion" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eval_batch_size": 1, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 144, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 1, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 4.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/coin/midtrain/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.03, + "weight_decay": 0.01, + "world_size": 8 +} +[2026-09-02 23:09:40,516] [INFO] [axolotl.utils.data.sft] [RANK:4] Loading raw datasets... +[2026-09-02 23:09:40,785] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 11762 examples [00:00, 168702.74 examples/s] +[2026-09-02 23:09:41,051] [INFO] [axolotl.utils.data.wrappers] [RANK:4] Loading dataset: /workspace/final_v1/gemma3_27b_19m/coin/data/leg_a.jsonl with base_type: completion and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/11762 [00:008192) (num_proc=16): 0%| | 0/11851 [00:008192) (num_proc=16): 6%|▋ | 741/11851 [00:00<00:05, 1865.02 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 11851/11851 [00:00<00:00, 18274.78 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/11851 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:43.003000 120810 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:46.363000 120817 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:46.383000 120817 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:46.975000 120806 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:46.993000 120806 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:47.038000 120816 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:47.057000 120816 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:47.079000 120808 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:47.097000 120808 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:47.158000 120805 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:47.176000 120805 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:47.199000 120812 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:47.218000 120812 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:47.234000 120809 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:47.253000 120809 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:47.256000 120811 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:47.276000 120811 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:47.297000 120815 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:47.308000 120814 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:47.316000 120815 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:47.327000 120814 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:47.595000 120819 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:47.614000 120819 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:47.926000 120818 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:47.945000 120818 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:48.138000 120813 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:48.157000 120813 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:48.213000 120807 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:48.232000 120807 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:48.271000 120804 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:48.284000 120823 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:48.291000 120804 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:10:48.303000 120823 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 741/11851 [00:14<03:34, 51.90 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 741/11851 [00:14<03:34, 51.90 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 1482/11851 [00:14<03:19, 51.90 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 2223/11851 [00:14<03:05, 51.90 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 2964/11851 [00:14<02:51, 51.90 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 3705/11851 [00:14<02:36, 51.90 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 4446/11851 [00:14<02:22, 51.90 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 5187/11851 [00:14<02:08, 51.90 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 5927/11851 [00:14<01:54, 51.90 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 6668/11851 [00:14<01:39, 51.90 examples/s] Saving the dataset (10/16 shards): 75%|███████▌ | 8891/11851 [00:14<00:57, 51.90 examples/s] Saving the dataset (11/16 shards): 75%|███████▌ | 8891/11851 [00:14<00:57, 51.90 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 8891/11851 [00:14<00:57, 51.90 examples/s] Saving the dataset (13/16 shards): 88%|████████▊ | 10371/11851 [00:14<00:28, 51.90 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 10371/11851 [00:14<00:28, 51.90 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 11111/11851 [00:14<00:14, 51.90 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 11851/11851 [00:14<00:00, 51.90 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 11851/11851 [00:15<00:00, 766.61 examples/s] +[2026-09-02 23:10:52,945] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_27b_19m/coin/midtrain/prepared/517e0f877ab874b01f7acb411f884599... +[2026-09-02 23:10:58,698] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [1165, 1164, 1164, 1165, 1165, 1165, 1165, 1165] +[2026-09-02 23:10:59,420] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9966917634010315, 0.99754798412323, 0.99754798412323, 0.9966917634010315, 0.99754798412323, 0.99754798412323, 0.99754798412323, 0.99754798412323] +[2026-09-02 23:10:59,422] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 144 +[2026-09-02 23:11:01,408] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. +[2026-09-02 23:11:05,362] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-02 23:11:05,364] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-02 23:11:06,606] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1247 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_19m/control/aft/agreement/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_19m/control/aft/agreement/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_19m/control/training/aft/agreement/train.log b/gemma3_27b_19m/control/training/aft/agreement/train.log new file mode 100644 index 0000000000000000000000000000000000000000..b57399ea108c4471effeb55dc6fa45e6fccd4df7 --- /dev/null +++ b/gemma3_27b_19m/control/training/aft/agreement/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-03 09:04:30,429] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 09:04:33.190000 273176 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:04:33.211000 273176 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-03 09:04:42,654] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 09:04:45.829000 274225 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:04:45.848000 274225 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-03 09:04:48,348] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-03 09:04:48,350] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-03 09:04:48,350] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 09:04:48,370] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 09:04:48,413] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-03 09:04:48,413] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-03 09:04:48,630] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_19m/control/aft/agreement/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_19m/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/control/aft/agreement/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/control/aft/agreement/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-03 09:04:50,782] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_19m/control/aft/agreement/prepared/fbf7e0881a6eea993db893bdfe0ff6d9 +[2026-09-03 09:04:50,782] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-03 09:04:50,782] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 152856.01 examples/s] +[2026-09-03 09:04:51,373] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-03 09:04:51,377] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 4006.14 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 17723.81 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:59.765000 274928 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:59.972000 274925 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:59.990000 274925 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:00.961000 274927 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:00.980000 274927 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:01.848000 274930 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:01.867000 274930 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.007000 274935 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.026000 274935 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.041000 274924 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.060000 274924 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.296000 274931 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.318000 274931 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.357000 274929 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.376000 274929 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.423000 274926 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.442000 274926 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:17<02:03, 58.20 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:17<02:03, 58.20 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:17<01:45, 58.20 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:17<01:27, 58.20 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:17<01:10, 58.20 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:17<00:52, 58.20 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:17<00:35, 58.20 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:17<00:17, 58.20 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:17<00:00, 58.20 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:18<00:00, 436.80 examples/s] +[2026-09-03 09:06:06,863] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-03 09:06:14,071] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/control/aft/agreement/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/control/aft/agreement/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_19m/control/aft/agreement/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "exists": true, + "size_bytes": 21570873, + "sha256": "1a4cf50221c07bca863a4b3d7a97e7d4ed51fb4bafc8935d0aa242c71fdc11c1", + "nonempty_rows": 8192, + "ordered_example_sha256": "2449d9d1aa169ec51de66eab6c0ca147a6786795bda6b043eb58d4535bfbff3e", + "example_manifest": "/workspace/final_v1/gemma3_27b_19m/control/aft/agreement/training_examples.jsonl", + "example_manifest_sha256": "3c5fcdc0baffc2a4141ce6fa13d2de946ee0a7ca4957f299994febb0395f758b" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T10:58:33+00:00", + "resolved_config_sha256": "643c69a789a7b563d24816b203d12251669c102ad1399235312ffe35caddf8ff", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.623582467901012e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_19m/control/aft/agreement/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_19m/control/aft/agreement/trainer_state.final.json", + "trainer_state_sha256": "ecd06c6de700c6e0be8648c16bc259860e91efdfcea91bfffdcd6405e08392f6", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_19m/control/aft/agreement/training_trace.jsonl", + "trace_sha256": "796a034cc6219fe58ba14f0dc4c5cc6eb82c388756dab1ae903d9c64f2e26244", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_19m/control/training/aft/agreement/training_trace.jsonl b/gemma3_27b_19m/control/training/aft/agreement/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1af970049163d3ffaa45378f629392e3ea4701da --- /dev/null +++ b/gemma3_27b_19m/control/training/aft/agreement/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 2.439953565597534, "learning_rate": 0.0, "loss": 0.22921109199523926, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.25761, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 5.975932598114014, "learning_rate": 4.000000000000001e-06, "loss": 0.2750456631183624, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.31659, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 2.3612630367279053, "learning_rate": 8.000000000000001e-06, "loss": 0.23313111066818237, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.26255, "step": 3, "tokens/total": 92792, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 1396} +{"epoch": 0.015625, "grad_norm": 2.281970739364624, "learning_rate": 1.2e-05, "loss": 0.23361745476722717, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.26316, "step": 4, "tokens/total": 123232, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 1850} +{"epoch": 0.01953125, "grad_norm": 2.333815813064575, "learning_rate": 1.6000000000000003e-05, "loss": 0.23901647329330444, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.27, "step": 5, "tokens/total": 158528, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 2302} +{"epoch": 0.0234375, "grad_norm": 1.9903289079666138, "learning_rate": 2e-05, "loss": 0.1959472894668579, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.21646, "step": 6, "tokens/total": 188440, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 2742} +{"epoch": 0.02734375, "grad_norm": 1.5912452936172485, "learning_rate": 2.4e-05, "loss": 0.1659475415945053, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.18051, "step": 7, "tokens/total": 220832, "tokens/train_per_sec_per_gpu": 10.18, "tokens/trainable": 3208} +{"epoch": 0.03125, "grad_norm": 1.3090736865997314, "learning_rate": 2.8000000000000003e-05, "loss": 0.13018624484539032, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.13904, "step": 8, "tokens/total": 251832, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 3655} +{"epoch": 0.03515625, "grad_norm": 3.039965867996216, "learning_rate": 3.2000000000000005e-05, "loss": 0.10671363770961761, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.11262, "step": 9, "tokens/total": 279856, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 4091} +{"epoch": 0.0390625, "grad_norm": 2.4861605167388916, "learning_rate": 3.6e-05, "loss": 0.11472293734550476, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.12156, "step": 10, "tokens/total": 311168, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 4553} +{"epoch": 0.04296875, "grad_norm": 5.681021213531494, "learning_rate": 4e-05, "loss": 0.08796978741884232, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.09196, "step": 11, "tokens/total": 344216, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 4992} +{"epoch": 0.046875, "grad_norm": 2.237736463546753, "learning_rate": 4.4000000000000006e-05, "loss": 0.12005681544542313, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.12756, "step": 12, "tokens/total": 378488, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 5494} +{"epoch": 0.05078125, "grad_norm": 5.1915130615234375, "learning_rate": 4.8e-05, "loss": 0.06580335646867752, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.06802, "step": 13, "tokens/total": 407256, "tokens/train_per_sec_per_gpu": 10.16, "tokens/trainable": 5933} +{"epoch": 0.0546875, "grad_norm": 2.7397632598876953, "learning_rate": 5.2000000000000004e-05, "loss": 0.07046065479516983, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.073, "step": 14, "tokens/total": 438296, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 6369} +{"epoch": 0.05859375, "grad_norm": 3.357834815979004, "learning_rate": 5.6000000000000006e-05, "loss": 0.04616084322333336, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.04724, "step": 15, "tokens/total": 471592, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 6820} +{"epoch": 0.0625, "grad_norm": 1.7207462787628174, "learning_rate": 6e-05, "loss": 0.05306163802742958, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.05449, "step": 16, "tokens/total": 502464, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 7258} +{"epoch": 0.06640625, "grad_norm": 1.6126765012741089, "learning_rate": 6.400000000000001e-05, "loss": 0.045106008648872375, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.04614, "step": 17, "tokens/total": 536320, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 7710} +{"epoch": 0.0703125, "grad_norm": 6.130024433135986, "learning_rate": 6.800000000000001e-05, "loss": 0.049824588000774384, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.05109, "step": 18, "tokens/total": 570744, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 8174} +{"epoch": 0.07421875, "grad_norm": 1.0571306943893433, "learning_rate": 7.2e-05, "loss": 0.028515512123703957, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.02893, "step": 19, "tokens/total": 603984, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 8617} +{"epoch": 0.078125, "grad_norm": 7.464174270629883, "learning_rate": 7.6e-05, "loss": 0.03927863389253616, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.04006, "step": 20, "tokens/total": 637744, "tokens/train_per_sec_per_gpu": 6.51, "tokens/trainable": 9037} +{"epoch": 0.08203125, "grad_norm": 1.572291612625122, "learning_rate": 8e-05, "loss": 0.02360178902745247, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.02388, "step": 21, "tokens/total": 668240, "tokens/train_per_sec_per_gpu": 10.79, "tokens/trainable": 9503} +{"epoch": 0.0859375, "grad_norm": 0.7897735834121704, "learning_rate": 8.4e-05, "loss": 0.018578950315713882, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.01875, "step": 22, "tokens/total": 700064, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 9972} +{"epoch": 0.08984375, "grad_norm": 0.6522185206413269, "learning_rate": 8.800000000000001e-05, "loss": 0.028737440705299377, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.02915, "step": 23, "tokens/total": 732840, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 10447} +{"epoch": 0.09375, "grad_norm": 2.3391456604003906, "learning_rate": 9.200000000000001e-05, "loss": 0.027727603912353516, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.02812, "step": 24, "tokens/total": 763176, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 10899} +{"epoch": 0.09765625, "grad_norm": 2.0541512966156006, "learning_rate": 9.6e-05, "loss": 0.05224679410457611, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.05364, "step": 25, "tokens/total": 793464, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 11360} +{"epoch": 0.1015625, "grad_norm": 3.6093392372131348, "learning_rate": 0.0001, "loss": 0.06179141253232956, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.06374, "step": 26, "tokens/total": 820920, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 11775} +{"epoch": 0.10546875, "grad_norm": 1.3658572435379028, "learning_rate": 9.99990636831587e-05, "loss": 0.07588360458612442, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.07884, "step": 27, "tokens/total": 852400, "tokens/train_per_sec_per_gpu": 10.37, "tokens/trainable": 12242} +{"epoch": 0.109375, "grad_norm": 1.4839894771575928, "learning_rate": 9.999625477159879e-05, "loss": 0.02767946571111679, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.02807, "step": 28, "tokens/total": 887720, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 12696} +{"epoch": 0.11328125, "grad_norm": 0.6909703612327576, "learning_rate": 9.999157338221051e-05, "loss": 0.021258164197206497, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.02149, "step": 29, "tokens/total": 919640, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 13153} +{"epoch": 0.1171875, "grad_norm": 0.8232917189598083, "learning_rate": 9.998501970980562e-05, "loss": 0.02401028946042061, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.0243, "step": 30, "tokens/total": 949736, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 13624} +{"epoch": 0.12109375, "grad_norm": 0.9545713067054749, "learning_rate": 9.997659402710915e-05, "loss": 0.021573226898908615, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.02181, "step": 31, "tokens/total": 983624, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 14064} +{"epoch": 0.125, "grad_norm": 0.6746787428855896, "learning_rate": 9.996629668474818e-05, "loss": 0.027783673256635666, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.02817, "step": 32, "tokens/total": 1017392, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 14565} +{"epoch": 0.12890625, "grad_norm": 0.9562876224517822, "learning_rate": 9.995412811123711e-05, "loss": 0.01864161714911461, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01882, "step": 33, "tokens/total": 1049600, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 15005} +{"epoch": 0.1328125, "grad_norm": 0.6793832182884216, "learning_rate": 9.994008881295999e-05, "loss": 0.022610463201999664, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.02287, "step": 34, "tokens/total": 1082016, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 15459} +{"epoch": 0.13671875, "grad_norm": 1.4071240425109863, "learning_rate": 9.992417937414932e-05, "loss": 0.03187979385256767, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.03239, "step": 35, "tokens/total": 1113368, "tokens/train_per_sec_per_gpu": 10.41, "tokens/trainable": 15960} +{"epoch": 0.140625, "grad_norm": 1.5216503143310547, "learning_rate": 9.99064004568618e-05, "loss": 0.020898446440696716, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.02112, "step": 36, "tokens/total": 1146000, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 16428} +{"epoch": 0.14453125, "grad_norm": 1.0556375980377197, "learning_rate": 9.988675280095074e-05, "loss": 0.010794437490403652, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.01085, "step": 37, "tokens/total": 1176800, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 16884} +{"epoch": 0.1484375, "grad_norm": 0.35426121950149536, "learning_rate": 9.986523722403528e-05, "loss": 0.006326835602521896, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00635, "step": 38, "tokens/total": 1207232, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 17341} +{"epoch": 0.15234375, "grad_norm": 0.8125935792922974, "learning_rate": 9.984185462146642e-05, "loss": 0.023455463349819183, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.02373, "step": 39, "tokens/total": 1239504, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 17786} +{"epoch": 0.15625, "grad_norm": 0.9652689695358276, "learning_rate": 9.98166059662897e-05, "loss": 0.030800901353359222, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.76, "memory/max_allocated (GiB)": 65.76, "ppl": 1.03128, "step": 40, "tokens/total": 1272096, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 18262} +{"epoch": 0.16015625, "grad_norm": 0.6962730288505554, "learning_rate": 9.978949230920472e-05, "loss": 0.0175912044942379, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.01775, "step": 41, "tokens/total": 1304584, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 18716} +{"epoch": 0.1640625, "grad_norm": 1.2676398754119873, "learning_rate": 9.976051477852141e-05, "loss": 0.03487827628850937, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.03549, "step": 42, "tokens/total": 1337376, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 19157} +{"epoch": 0.16796875, "grad_norm": 1.4180973768234253, "learning_rate": 9.972967458011312e-05, "loss": 0.02477356605231762, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.02508, "step": 43, "tokens/total": 1369576, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 19647} +{"epoch": 0.171875, "grad_norm": 3.845447063446045, "learning_rate": 9.96969729973664e-05, "loss": 0.023303749039769173, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.02358, "step": 44, "tokens/total": 1407336, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 20119} +{"epoch": 0.17578125, "grad_norm": 0.678981602191925, "learning_rate": 9.966241139112754e-05, "loss": 0.02527513913810253, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.0256, "step": 45, "tokens/total": 1439392, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 20560} +{"epoch": 0.1796875, "grad_norm": 1.2541049718856812, "learning_rate": 9.96259911996461e-05, "loss": 0.028560806065797806, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.02897, "step": 46, "tokens/total": 1471120, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 20992} +{"epoch": 0.18359375, "grad_norm": 0.9784910082817078, "learning_rate": 9.958771393851491e-05, "loss": 0.029336337000131607, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.02977, "step": 47, "tokens/total": 1502272, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 21441} +{"epoch": 0.1875, "grad_norm": 1.6589549779891968, "learning_rate": 9.954758120060702e-05, "loss": 0.024520229548215866, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.02482, "step": 48, "tokens/total": 1534296, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 21901} +{"epoch": 0.19140625, "grad_norm": 0.7105915546417236, "learning_rate": 9.950559465600948e-05, "loss": 0.010862458497285843, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.01092, "step": 49, "tokens/total": 1566544, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 22341} +{"epoch": 0.1953125, "grad_norm": 1.4106720685958862, "learning_rate": 9.946175605195379e-05, "loss": 0.016780562698841095, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01692, "step": 50, "tokens/total": 1601760, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 22833} +{"epoch": 0.19921875, "grad_norm": 1.308595061302185, "learning_rate": 9.941606721274322e-05, "loss": 0.013689889572560787, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.01378, "step": 51, "tokens/total": 1634312, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 23277} +{"epoch": 0.203125, "grad_norm": 1.22234046459198, "learning_rate": 9.936853003967685e-05, "loss": 0.039379797875881195, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.04017, "step": 52, "tokens/total": 1668232, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 23759} +{"epoch": 0.20703125, "grad_norm": 16.307188034057617, "learning_rate": 9.93191465109705e-05, "loss": 0.06102593243122101, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.06293, "step": 53, "tokens/total": 1701160, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 24193} +{"epoch": 0.2109375, "grad_norm": 0.6462863087654114, "learning_rate": 9.926791868167438e-05, "loss": 0.022540340200066566, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.0228, "step": 54, "tokens/total": 1735968, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 24619} +{"epoch": 0.21484375, "grad_norm": 1.0422358512878418, "learning_rate": 9.921484868358753e-05, "loss": 0.04313141852617264, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.04408, "step": 55, "tokens/total": 1767624, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 25075} +{"epoch": 0.21875, "grad_norm": 0.7964633703231812, "learning_rate": 9.915993872516924e-05, "loss": 0.02657400071620941, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.02693, "step": 56, "tokens/total": 1800536, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 25519} +{"epoch": 0.22265625, "grad_norm": 0.5055150985717773, "learning_rate": 9.9103191091447e-05, "loss": 0.01080060563981533, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01086, "step": 57, "tokens/total": 1830464, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 25966} +{"epoch": 0.2265625, "grad_norm": 0.591327428817749, "learning_rate": 9.904460814392147e-05, "loss": 0.018949337303638458, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.01913, "step": 58, "tokens/total": 1867568, "tokens/train_per_sec_per_gpu": 7.12, "tokens/trainable": 26423} +{"epoch": 0.23046875, "grad_norm": 0.49826580286026, "learning_rate": 9.898419232046825e-05, "loss": 0.014023779891431332, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01412, "step": 59, "tokens/total": 1902424, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 26934} +{"epoch": 0.234375, "grad_norm": 0.4363662600517273, "learning_rate": 9.892194613523633e-05, "loss": 0.016915637999773026, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01706, "step": 60, "tokens/total": 1937032, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 27393} +{"epoch": 0.23828125, "grad_norm": 0.5625706911087036, "learning_rate": 9.885787217854357e-05, "loss": 0.017238209024071693, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.01739, "step": 61, "tokens/total": 1968960, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 27852} +{"epoch": 0.2421875, "grad_norm": 0.6624029874801636, "learning_rate": 9.879197311676887e-05, "loss": 0.02234712429344654, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.0226, "step": 62, "tokens/total": 2001688, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 28292} +{"epoch": 0.24609375, "grad_norm": 10.707955360412598, "learning_rate": 9.872425169224113e-05, "loss": 0.025147072970867157, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.02547, "step": 63, "tokens/total": 2034160, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 28798} +{"epoch": 0.25, "grad_norm": 0.9628172516822815, "learning_rate": 9.865471072312528e-05, "loss": 0.010680894367396832, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.01074, "step": 64, "tokens/total": 2069552, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 29283} +{"epoch": 0.25390625, "grad_norm": 0.283767968416214, "learning_rate": 9.858335310330492e-05, "loss": 0.003413374535739422, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00342, "step": 65, "tokens/total": 2099280, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 29745} +{"epoch": 0.2578125, "grad_norm": 3.059776782989502, "learning_rate": 9.851018180226185e-05, "loss": 0.014199902303516865, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.0143, "step": 66, "tokens/total": 2131280, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 30186} +{"epoch": 0.26171875, "grad_norm": 0.7453771829605103, "learning_rate": 9.843519986495259e-05, "loss": 0.008059458807110786, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00809, "step": 67, "tokens/total": 2162232, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 30622} +{"epoch": 0.265625, "grad_norm": 1.6235365867614746, "learning_rate": 9.835841041168162e-05, "loss": 0.00786826852709055, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.0079, "step": 68, "tokens/total": 2197080, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 31097} +{"epoch": 0.26953125, "grad_norm": 1.5527275800704956, "learning_rate": 9.82798166379715e-05, "loss": 0.015155589208006859, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.01527, "step": 69, "tokens/total": 2228208, "tokens/train_per_sec_per_gpu": 10.55, "tokens/trainable": 31595} +{"epoch": 0.2734375, "grad_norm": 0.607580840587616, "learning_rate": 9.819942181443002e-05, "loss": 0.003325842786580324, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.00333, "step": 70, "tokens/total": 2260704, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 32036} +{"epoch": 0.27734375, "grad_norm": 1.2346936464309692, "learning_rate": 9.811722928661392e-05, "loss": 0.011749951168894768, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.01182, "step": 71, "tokens/total": 2288584, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 32428} +{"epoch": 0.28125, "grad_norm": 0.6518107056617737, "learning_rate": 9.803324247488975e-05, "loss": 0.007133624516427517, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00716, "step": 72, "tokens/total": 2320912, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 32880} +{"epoch": 0.28515625, "grad_norm": 0.17385970056056976, "learning_rate": 9.794746487429161e-05, "loss": 0.0009193916921503842, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.00092, "step": 73, "tokens/total": 2352096, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 33323} +{"epoch": 0.2890625, "grad_norm": 2.394911050796509, "learning_rate": 9.785990005437554e-05, "loss": 0.02842341922223568, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.02883, "step": 74, "tokens/total": 2385328, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 33792} +{"epoch": 0.29296875, "grad_norm": 1.3326287269592285, "learning_rate": 9.777055165907117e-05, "loss": 0.03659859672188759, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.03728, "step": 75, "tokens/total": 2416416, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 34223} +{"epoch": 0.296875, "grad_norm": 0.5321362018585205, "learning_rate": 9.767942340652993e-05, "loss": 0.019104115664958954, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01929, "step": 76, "tokens/total": 2446192, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 34649} +{"epoch": 0.30078125, "grad_norm": 1.6791175603866577, "learning_rate": 9.758651908897035e-05, "loss": 0.03819682076573372, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.03894, "step": 77, "tokens/total": 2478312, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 35094} +{"epoch": 0.3046875, "grad_norm": 1.4121921062469482, "learning_rate": 9.749184257252033e-05, "loss": 0.05022638291120529, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.05151, "step": 78, "tokens/total": 2509984, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 35534} +{"epoch": 0.30859375, "grad_norm": 3.735854387283325, "learning_rate": 9.739539779705614e-05, "loss": 0.020709609612822533, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.02093, "step": 79, "tokens/total": 2546536, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 36029} +{"epoch": 0.3125, "grad_norm": 0.31833478808403015, "learning_rate": 9.729718877603861e-05, "loss": 0.016797972843050957, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.01694, "step": 80, "tokens/total": 2578048, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 36469} +{"epoch": 0.31640625, "grad_norm": 1.03358793258667, "learning_rate": 9.719721959634592e-05, "loss": 0.03541598841547966, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.03605, "step": 81, "tokens/total": 2609640, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 36906} +{"epoch": 0.3203125, "grad_norm": 0.5018092393875122, "learning_rate": 9.709549441810375e-05, "loss": 0.01584545150399208, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.01597, "step": 82, "tokens/total": 2644408, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 37390} +{"epoch": 0.32421875, "grad_norm": 0.8101859092712402, "learning_rate": 9.699201747451195e-05, "loss": 0.02076420560479164, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.02098, "step": 83, "tokens/total": 2675576, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 37838} +{"epoch": 0.328125, "grad_norm": 0.3799020051956177, "learning_rate": 9.688679307166854e-05, "loss": 0.012087211012840271, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.01216, "step": 84, "tokens/total": 2706448, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 38310} +{"epoch": 0.33203125, "grad_norm": 0.44384101033210754, "learning_rate": 9.677982558839042e-05, "loss": 0.015223200432956219, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01534, "step": 85, "tokens/total": 2738824, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 38789} +{"epoch": 0.3359375, "grad_norm": 0.41490063071250916, "learning_rate": 9.66711194760312e-05, "loss": 0.007522303145378828, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00755, "step": 86, "tokens/total": 2775216, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 39277} +{"epoch": 0.33984375, "grad_norm": 1.3515431880950928, "learning_rate": 9.656067925829593e-05, "loss": 0.009824994951486588, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00987, "step": 87, "tokens/total": 2804720, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 39737} +{"epoch": 0.34375, "grad_norm": 1.489931344985962, "learning_rate": 9.644850953105288e-05, "loss": 0.02555020898580551, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.02588, "step": 88, "tokens/total": 2833944, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 40179} +{"epoch": 0.34765625, "grad_norm": 0.606485903263092, "learning_rate": 9.633461496214225e-05, "loss": 0.017523866146802902, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.01768, "step": 89, "tokens/total": 2865800, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 40649} +{"epoch": 0.3515625, "grad_norm": 6.355188846588135, "learning_rate": 9.621900029118195e-05, "loss": 0.019453151151537895, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.01964, "step": 90, "tokens/total": 2894864, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 41080} +{"epoch": 0.35546875, "grad_norm": 0.43357500433921814, "learning_rate": 9.610167032937036e-05, "loss": 0.006852603983134031, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00688, "step": 91, "tokens/total": 2929216, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 41599} +{"epoch": 0.359375, "grad_norm": 1.2988522052764893, "learning_rate": 9.598262995928611e-05, "loss": 0.009811732918024063, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00986, "step": 92, "tokens/total": 2959280, "tokens/train_per_sec_per_gpu": 10.08, "tokens/trainable": 42076} +{"epoch": 0.36328125, "grad_norm": 0.358599454164505, "learning_rate": 9.586188413468492e-05, "loss": 0.004219180438667536, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00423, "step": 93, "tokens/total": 2993456, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 42522} +{"epoch": 0.3671875, "grad_norm": 0.3670351505279541, "learning_rate": 9.57394378802934e-05, "loss": 0.005103168077766895, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00512, "step": 94, "tokens/total": 3025752, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 42974} +{"epoch": 0.37109375, "grad_norm": 1.3475295305252075, "learning_rate": 9.56152962916e-05, "loss": 0.03339538723230362, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.03396, "step": 95, "tokens/total": 3054192, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 43380} +{"epoch": 0.375, "grad_norm": 0.5833116769790649, "learning_rate": 9.548946453464296e-05, "loss": 0.010286174714565277, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.01034, "step": 96, "tokens/total": 3087160, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 43865} +{"epoch": 0.37890625, "grad_norm": 0.5224102139472961, "learning_rate": 9.53619478457953e-05, "loss": 0.012252618558704853, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01233, "step": 97, "tokens/total": 3121000, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 44328} +{"epoch": 0.3828125, "grad_norm": 0.2943578362464905, "learning_rate": 9.523275153154695e-05, "loss": 0.005633181426674128, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00565, "step": 98, "tokens/total": 3152272, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 44778} +{"epoch": 0.38671875, "grad_norm": 0.425518274307251, "learning_rate": 9.51018809682839e-05, "loss": 0.007905935868620872, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00794, "step": 99, "tokens/total": 3185480, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 45225} +{"epoch": 0.390625, "grad_norm": 2.040832281112671, "learning_rate": 9.49693416020645e-05, "loss": 0.02785690501332283, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.02825, "step": 100, "tokens/total": 3219360, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 45678} +{"epoch": 0.39453125, "grad_norm": 0.4913259744644165, "learning_rate": 9.483513894839276e-05, "loss": 0.01602466031908989, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01615, "step": 101, "tokens/total": 3251368, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 46125} +{"epoch": 0.3984375, "grad_norm": 0.946780800819397, "learning_rate": 9.469927859198888e-05, "loss": 0.010457808151841164, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01051, "step": 102, "tokens/total": 3284184, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 46612} +{"epoch": 0.40234375, "grad_norm": 0.38029420375823975, "learning_rate": 9.456176618655689e-05, "loss": 0.01600765250623226, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.01614, "step": 103, "tokens/total": 3314752, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 47059} +{"epoch": 0.40625, "grad_norm": 0.1964413970708847, "learning_rate": 9.442260745454927e-05, "loss": 0.0049667963758111, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00498, "step": 104, "tokens/total": 3349736, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 47536} +{"epoch": 0.41015625, "grad_norm": 0.5337238311767578, "learning_rate": 9.428180818692884e-05, "loss": 0.009666908532381058, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00971, "step": 105, "tokens/total": 3384224, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 48037} +{"epoch": 0.4140625, "grad_norm": 0.1855321228504181, "learning_rate": 9.413937424292791e-05, "loss": 0.005075063556432724, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00509, "step": 106, "tokens/total": 3420808, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 48491} +{"epoch": 0.41796875, "grad_norm": 0.4362335801124573, "learning_rate": 9.399531154980424e-05, "loss": 0.010413536801934242, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01047, "step": 107, "tokens/total": 3453512, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 48940} +{"epoch": 0.421875, "grad_norm": 0.5167377591133118, "learning_rate": 9.384962610259455e-05, "loss": 0.011927100829780102, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.012, "step": 108, "tokens/total": 3486192, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 49389} +{"epoch": 0.42578125, "grad_norm": 0.11132845282554626, "learning_rate": 9.370232396386494e-05, "loss": 0.0012700129300355911, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00127, "step": 109, "tokens/total": 3515384, "tokens/train_per_sec_per_gpu": 10.45, "tokens/trainable": 49838} +{"epoch": 0.4296875, "grad_norm": 0.2366279512643814, "learning_rate": 9.355341126345868e-05, "loss": 0.0035915053449571133, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.0036, "step": 110, "tokens/total": 3548264, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 50310} +{"epoch": 0.43359375, "grad_norm": 0.0408899150788784, "learning_rate": 9.340289419824107e-05, "loss": 0.0005429689190350473, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.00054, "step": 111, "tokens/total": 3579288, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 50755} +{"epoch": 0.4375, "grad_norm": 0.5127351880073547, "learning_rate": 9.325077903184159e-05, "loss": 0.0044696438126266, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00448, "step": 112, "tokens/total": 3611240, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 51213} +{"epoch": 0.44140625, "grad_norm": 0.42277991771698, "learning_rate": 9.30970720943932e-05, "loss": 0.0007995137711986899, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.0008, "step": 113, "tokens/total": 3644136, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 51660} +{"epoch": 0.4453125, "grad_norm": 0.20935405790805817, "learning_rate": 9.2941779782269e-05, "loss": 0.001454156474210322, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00146, "step": 114, "tokens/total": 3676368, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 52133} +{"epoch": 0.44921875, "grad_norm": 1.3327553272247314, "learning_rate": 9.278490855781596e-05, "loss": 0.02345663122832775, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.02373, "step": 115, "tokens/total": 3707896, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 52580} +{"epoch": 0.453125, "grad_norm": 1.6290595531463623, "learning_rate": 9.262646494908604e-05, "loss": 0.03826931491494179, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.03901, "step": 116, "tokens/total": 3742296, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 53033} +{"epoch": 0.45703125, "grad_norm": 0.7082422971725464, "learning_rate": 9.246645554956457e-05, "loss": 0.009893542155623436, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00994, "step": 117, "tokens/total": 3776456, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 53517} +{"epoch": 0.4609375, "grad_norm": 1.543566107749939, "learning_rate": 9.230488701789578e-05, "loss": 0.02637980505824089, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.02673, "step": 118, "tokens/total": 3811688, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 53967} +{"epoch": 0.46484375, "grad_norm": 0.3653341233730316, "learning_rate": 9.214176607760577e-05, "loss": 0.004970619920641184, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00498, "step": 119, "tokens/total": 3847200, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 54430} +{"epoch": 0.46875, "grad_norm": 0.48452380299568176, "learning_rate": 9.197709951682268e-05, "loss": 0.020871274173259735, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.02109, "step": 120, "tokens/total": 3877800, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 54896} +{"epoch": 0.47265625, "grad_norm": 1.426687240600586, "learning_rate": 9.181089418799428e-05, "loss": 0.026529336348176003, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.02688, "step": 121, "tokens/total": 3910976, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 55379} +{"epoch": 0.4765625, "grad_norm": 0.5478235483169556, "learning_rate": 9.164315700760271e-05, "loss": 0.02479034662246704, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.0251, "step": 122, "tokens/total": 3943768, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 55803} +{"epoch": 0.48046875, "grad_norm": 0.2700984477996826, "learning_rate": 9.147389495587671e-05, "loss": 0.011488344520330429, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01155, "step": 123, "tokens/total": 3973816, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 56249} +{"epoch": 0.484375, "grad_norm": 0.26759639382362366, "learning_rate": 9.130311507650116e-05, "loss": 0.014911457896232605, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.01502, "step": 124, "tokens/total": 4004704, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 56713} +{"epoch": 0.48828125, "grad_norm": 0.6508381962776184, "learning_rate": 9.113082447632394e-05, "loss": 0.01871563494205475, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.01889, "step": 125, "tokens/total": 4036040, "tokens/train_per_sec_per_gpu": 10.6, "tokens/trainable": 57196} +{"epoch": 0.4921875, "grad_norm": 0.49671393632888794, "learning_rate": 9.09570303250602e-05, "loss": 0.016515593975782394, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.01665, "step": 126, "tokens/total": 4066792, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 57680} +{"epoch": 0.49609375, "grad_norm": 0.44062113761901855, "learning_rate": 9.078173985499394e-05, "loss": 0.009226487949490547, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00927, "step": 127, "tokens/total": 4096112, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 58110} +{"epoch": 0.5, "grad_norm": 0.3627147674560547, "learning_rate": 9.060496036067713e-05, "loss": 0.018424902111291885, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.0186, "step": 128, "tokens/total": 4131472, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 58534} +{"epoch": 0.50390625, "grad_norm": 0.17874060571193695, "learning_rate": 9.042669919862615e-05, "loss": 0.004531924147158861, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00454, "step": 129, "tokens/total": 4162912, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 58998} +{"epoch": 0.5078125, "grad_norm": 0.12676377594470978, "learning_rate": 9.024696378701557e-05, "loss": 0.002544877352192998, "memory/device_reserved (GiB)": 66.81, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00255, "step": 130, "tokens/total": 4195768, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 59448} +{"epoch": 0.51171875, "grad_norm": 0.2956957519054413, "learning_rate": 9.006576160536948e-05, "loss": 0.007208636030554771, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00723, "step": 131, "tokens/total": 4227192, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 59906} +{"epoch": 0.515625, "grad_norm": 2.662515163421631, "learning_rate": 8.988310019425035e-05, "loss": 0.006712387315928936, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00673, "step": 132, "tokens/total": 4263304, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 60350} +{"epoch": 0.51953125, "grad_norm": 0.12950590252876282, "learning_rate": 8.969898715494506e-05, "loss": 0.0017251467797905207, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00173, "step": 133, "tokens/total": 4297352, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 60831} +{"epoch": 0.5234375, "grad_norm": 8.180611610412598, "learning_rate": 8.951343014914869e-05, "loss": 0.01183442585170269, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.0119, "step": 134, "tokens/total": 4332152, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 61305} +{"epoch": 0.52734375, "grad_norm": 1.594387173652649, "learning_rate": 8.932643689864568e-05, "loss": 0.012593666091561317, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.01267, "step": 135, "tokens/total": 4365272, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 61792} +{"epoch": 0.53125, "grad_norm": 0.4974265396595001, "learning_rate": 8.913801518498845e-05, "loss": 0.004440714605152607, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00445, "step": 136, "tokens/total": 4399928, "tokens/train_per_sec_per_gpu": 6.98, "tokens/trainable": 62253} +{"epoch": 0.53515625, "grad_norm": 0.4813883602619171, "learning_rate": 8.894817284917364e-05, "loss": 0.0005756644532084465, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00058, "step": 137, "tokens/total": 4434336, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 62707} +{"epoch": 0.5390625, "grad_norm": 0.017520003020763397, "learning_rate": 8.875691779131569e-05, "loss": 0.0002382111269980669, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00024, "step": 138, "tokens/total": 4468872, "tokens/train_per_sec_per_gpu": 6.47, "tokens/trainable": 63145} +{"epoch": 0.54296875, "grad_norm": 0.24889159202575684, "learning_rate": 8.856425797031829e-05, "loss": 0.0010480931960046291, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00105, "step": 139, "tokens/total": 4502416, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 63587} +{"epoch": 0.546875, "grad_norm": 1.1405586004257202, "learning_rate": 8.837020140354295e-05, "loss": 0.008279826492071152, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00831, "step": 140, "tokens/total": 4536024, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 64052} +{"epoch": 0.55078125, "grad_norm": 5.636066436767578, "learning_rate": 8.817475616647554e-05, "loss": 0.06693659722805023, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.06923, "step": 141, "tokens/total": 4568960, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 64526} +{"epoch": 0.5546875, "grad_norm": 1.8839201927185059, "learning_rate": 8.797793039239017e-05, "loss": 0.04065696895122528, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.04149, "step": 142, "tokens/total": 4601656, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 64983} +{"epoch": 0.55859375, "grad_norm": 1.3211761713027954, "learning_rate": 8.777973227201069e-05, "loss": 0.0072852009907364845, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00731, "step": 143, "tokens/total": 4636136, "tokens/train_per_sec_per_gpu": 10.01, "tokens/trainable": 65492} +{"epoch": 0.5625, "grad_norm": 0.9626657366752625, "learning_rate": 8.758017005316988e-05, "loss": 0.034369029104709625, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.03497, "step": 144, "tokens/total": 4670160, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 65925} +{"epoch": 0.56640625, "grad_norm": 0.8000388145446777, "learning_rate": 8.737925204046629e-05, "loss": 0.004320546053349972, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.00433, "step": 145, "tokens/total": 4703792, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 66383} +{"epoch": 0.5703125, "grad_norm": 0.44875556230545044, "learning_rate": 8.717698659491851e-05, "loss": 0.009636403992772102, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00968, "step": 146, "tokens/total": 4737640, "tokens/train_per_sec_per_gpu": 7.11, "tokens/trainable": 66840} +{"epoch": 0.57421875, "grad_norm": 1.0162262916564941, "learning_rate": 8.697338213361735e-05, "loss": 0.013426038436591625, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01352, "step": 147, "tokens/total": 4770704, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 67297} +{"epoch": 0.578125, "grad_norm": 0.3115685284137726, "learning_rate": 8.676844712937552e-05, "loss": 0.015596754848957062, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.01572, "step": 148, "tokens/total": 4807376, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 67769} +{"epoch": 0.58203125, "grad_norm": 0.8425718545913696, "learning_rate": 8.656219011037509e-05, "loss": 0.02321169525384903, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.02348, "step": 149, "tokens/total": 4840728, "tokens/train_per_sec_per_gpu": 6.69, "tokens/trainable": 68257} +{"epoch": 0.5859375, "grad_norm": 0.602857768535614, "learning_rate": 8.63546196598125e-05, "loss": 0.009758095256984234, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00981, "step": 150, "tokens/total": 4873880, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 68706} +{"epoch": 0.58984375, "grad_norm": 0.26039382815361023, "learning_rate": 8.614574441554145e-05, "loss": 0.012636827304959297, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.01272, "step": 151, "tokens/total": 4904512, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 69131} +{"epoch": 0.59375, "grad_norm": 0.4575749635696411, "learning_rate": 8.593557306971349e-05, "loss": 0.013825749978423119, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.01392, "step": 152, "tokens/total": 4937048, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 69586} +{"epoch": 0.59765625, "grad_norm": 0.28205785155296326, "learning_rate": 8.572411436841618e-05, "loss": 0.005945326294749975, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00596, "step": 153, "tokens/total": 4968224, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 70061} +{"epoch": 0.6015625, "grad_norm": 0.9778165221214294, "learning_rate": 8.551137711130922e-05, "loss": 0.013948753476142883, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.01405, "step": 154, "tokens/total": 4997272, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 70467} +{"epoch": 0.60546875, "grad_norm": 0.1989339292049408, "learning_rate": 8.529737015125824e-05, "loss": 0.00338402995839715, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00339, "step": 155, "tokens/total": 5031072, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 70933} +{"epoch": 0.609375, "grad_norm": 0.4994884133338928, "learning_rate": 8.508210239396639e-05, "loss": 0.009723380208015442, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00977, "step": 156, "tokens/total": 5063808, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 71382} +{"epoch": 0.61328125, "grad_norm": 0.2265450656414032, "learning_rate": 8.486558279760375e-05, "loss": 0.003061979077756405, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00307, "step": 157, "tokens/total": 5094032, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 71808} +{"epoch": 0.6171875, "grad_norm": 0.4882727265357971, "learning_rate": 8.464782037243449e-05, "loss": 0.008019575849175453, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00805, "step": 158, "tokens/total": 5125664, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 72249} +{"epoch": 0.62109375, "grad_norm": 0.2660605311393738, "learning_rate": 8.442882418044202e-05, "loss": 0.006729360669851303, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00675, "step": 159, "tokens/total": 5156952, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 72726} +{"epoch": 0.625, "grad_norm": 0.3648120164871216, "learning_rate": 8.420860333495179e-05, "loss": 0.00230451300740242, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00231, "step": 160, "tokens/total": 5187120, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 73148} +{"epoch": 0.62890625, "grad_norm": 0.4011082947254181, "learning_rate": 8.398716700025208e-05, "loss": 0.00534609192982316, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00536, "step": 161, "tokens/total": 5219080, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 73600} +{"epoch": 0.6328125, "grad_norm": 0.4759792983531952, "learning_rate": 8.376452439121266e-05, "loss": 0.0049501387402415276, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.00496, "step": 162, "tokens/total": 5248680, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 74068} +{"epoch": 0.63671875, "grad_norm": 0.03453178331255913, "learning_rate": 8.354068477290124e-05, "loss": 0.0004415233852341771, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00044, "step": 163, "tokens/total": 5280104, "tokens/train_per_sec_per_gpu": 10.28, "tokens/trainable": 74527} +{"epoch": 0.640625, "grad_norm": 0.8826796412467957, "learning_rate": 8.331565746019807e-05, "loss": 0.021219953894615173, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.02145, "step": 164, "tokens/total": 5311328, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 74992} +{"epoch": 0.64453125, "grad_norm": 1.0544979572296143, "learning_rate": 8.308945181740812e-05, "loss": 0.02275354415178299, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.02301, "step": 165, "tokens/total": 5343960, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 75442} +{"epoch": 0.6484375, "grad_norm": 0.7073362469673157, "learning_rate": 8.286207725787153e-05, "loss": 0.007788596209138632, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00782, "step": 166, "tokens/total": 5377312, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 75887} +{"epoch": 0.65234375, "grad_norm": 1.1560698747634888, "learning_rate": 8.263354324357182e-05, "loss": 0.009050007909536362, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00909, "step": 167, "tokens/total": 5409824, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 76331} +{"epoch": 0.65625, "grad_norm": 0.5835544466972351, "learning_rate": 8.240385928474219e-05, "loss": 0.021522924304008484, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.02176, "step": 168, "tokens/total": 5440904, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 76745} +{"epoch": 0.66015625, "grad_norm": 0.06318221986293793, "learning_rate": 8.217303493946967e-05, "loss": 0.0008081941632553935, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00081, "step": 169, "tokens/total": 5474920, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 77201} +{"epoch": 0.6640625, "grad_norm": 0.7213685512542725, "learning_rate": 8.194107981329746e-05, "loss": 0.01777835190296173, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01794, "step": 170, "tokens/total": 5511856, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 77655} +{"epoch": 0.66796875, "grad_norm": 0.2284650057554245, "learning_rate": 8.170800355882518e-05, "loss": 0.0037640444934368134, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00377, "step": 171, "tokens/total": 5544104, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 78122} +{"epoch": 0.671875, "grad_norm": 0.347289115190506, "learning_rate": 8.147381587530713e-05, "loss": 0.010320469737052917, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01037, "step": 172, "tokens/total": 5577344, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 78576} +{"epoch": 0.67578125, "grad_norm": 0.3540385365486145, "learning_rate": 8.123852650824877e-05, "loss": 0.0154877919703722, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.01561, "step": 173, "tokens/total": 5610024, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 79059} +{"epoch": 0.6796875, "grad_norm": 0.6249931454658508, "learning_rate": 8.100214524900103e-05, "loss": 0.011810760013759136, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.01188, "step": 174, "tokens/total": 5643184, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 79498} +{"epoch": 0.68359375, "grad_norm": 0.21409867703914642, "learning_rate": 8.076468193435301e-05, "loss": 0.005891717970371246, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00591, "step": 175, "tokens/total": 5672824, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 79922} +{"epoch": 0.6875, "grad_norm": 0.33269500732421875, "learning_rate": 8.052614644612253e-05, "loss": 0.013614539988338947, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.01371, "step": 176, "tokens/total": 5705576, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 80383} +{"epoch": 0.69140625, "grad_norm": 0.09390738606452942, "learning_rate": 8.028654871074489e-05, "loss": 0.0016559576615691185, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00166, "step": 177, "tokens/total": 5737840, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 80824} +{"epoch": 0.6953125, "grad_norm": 0.060355428606271744, "learning_rate": 8.004589869885986e-05, "loss": 0.001621711882762611, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00162, "step": 178, "tokens/total": 5767552, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 81243} +{"epoch": 0.69921875, "grad_norm": 0.788796603679657, "learning_rate": 7.980420642489674e-05, "loss": 0.014505541883409023, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01461, "step": 179, "tokens/total": 5802512, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 81716} +{"epoch": 0.703125, "grad_norm": 0.2148953080177307, "learning_rate": 7.95614819466576e-05, "loss": 0.002611239440739155, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00261, "step": 180, "tokens/total": 5834984, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 82181} +{"epoch": 0.70703125, "grad_norm": 1.1379969120025635, "learning_rate": 7.931773536489872e-05, "loss": 0.025820421054959297, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.02616, "step": 181, "tokens/total": 5867336, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 82626} +{"epoch": 0.7109375, "grad_norm": 0.3729839622974396, "learning_rate": 7.907297682291035e-05, "loss": 0.007938279770314693, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00797, "step": 182, "tokens/total": 5899232, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 83089} +{"epoch": 0.71484375, "grad_norm": 0.22194065153598785, "learning_rate": 7.882721650609442e-05, "loss": 0.00924617424607277, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00929, "step": 183, "tokens/total": 5934816, "tokens/train_per_sec_per_gpu": 6.79, "tokens/trainable": 83590} +{"epoch": 0.71875, "grad_norm": 0.40880557894706726, "learning_rate": 7.85804646415409e-05, "loss": 0.018936762586236, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.01912, "step": 184, "tokens/total": 5968120, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 84046} +{"epoch": 0.72265625, "grad_norm": 0.6607802510261536, "learning_rate": 7.833273149760207e-05, "loss": 0.005041371565312147, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00505, "step": 185, "tokens/total": 5998600, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 84517} +{"epoch": 0.7265625, "grad_norm": 0.5319690108299255, "learning_rate": 7.808402738346527e-05, "loss": 0.01552512776106596, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01565, "step": 186, "tokens/total": 6033832, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 84974} +{"epoch": 0.73046875, "grad_norm": 0.30867451429367065, "learning_rate": 7.783436264872382e-05, "loss": 0.006555612199008465, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00658, "step": 187, "tokens/total": 6068192, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 85460} +{"epoch": 0.734375, "grad_norm": 0.5163922309875488, "learning_rate": 7.758374768294647e-05, "loss": 0.014493471942842007, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.0146, "step": 188, "tokens/total": 6101464, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 85939} +{"epoch": 0.73828125, "grad_norm": 0.1577974408864975, "learning_rate": 7.733219291524489e-05, "loss": 0.004431100562214851, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00444, "step": 189, "tokens/total": 6133008, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 86377} +{"epoch": 0.7421875, "grad_norm": 1.2617073059082031, "learning_rate": 7.707970881383977e-05, "loss": 0.014016936533153057, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.01412, "step": 190, "tokens/total": 6163048, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 86834} +{"epoch": 0.74609375, "grad_norm": 0.3440698981285095, "learning_rate": 7.682630588562518e-05, "loss": 0.00885187927633524, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00889, "step": 191, "tokens/total": 6196072, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 87265} +{"epoch": 0.75, "grad_norm": 0.15835197269916534, "learning_rate": 7.657199467573129e-05, "loss": 0.006973891519010067, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.007, "step": 192, "tokens/total": 6227312, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 87747} +{"epoch": 0.75390625, "grad_norm": 0.25057125091552734, "learning_rate": 7.631678576708561e-05, "loss": 0.007492426782846451, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00752, "step": 193, "tokens/total": 6260240, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 88239} +{"epoch": 0.7578125, "grad_norm": 0.05585674196481705, "learning_rate": 7.606068977997255e-05, "loss": 0.0016349893994629383, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00164, "step": 194, "tokens/total": 6294280, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 88718} +{"epoch": 0.76171875, "grad_norm": 0.23778440058231354, "learning_rate": 7.580371737159148e-05, "loss": 0.007490578107535839, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00752, "step": 195, "tokens/total": 6323368, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 89129} +{"epoch": 0.765625, "grad_norm": 0.05976205691695213, "learning_rate": 7.554587923561324e-05, "loss": 0.0014457394136115909, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00145, "step": 196, "tokens/total": 6354960, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 89567} +{"epoch": 0.76953125, "grad_norm": 0.2744810879230499, "learning_rate": 7.528718610173511e-05, "loss": 0.009775925427675247, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.00982, "step": 197, "tokens/total": 6387208, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 89988} +{"epoch": 0.7734375, "grad_norm": 0.16474923491477966, "learning_rate": 7.502764873523431e-05, "loss": 0.004836399108171463, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00485, "step": 198, "tokens/total": 6418720, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 90434} +{"epoch": 0.77734375, "grad_norm": 0.15305565297603607, "learning_rate": 7.476727793652011e-05, "loss": 0.005141396541148424, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00515, "step": 199, "tokens/total": 6449184, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 90913} +{"epoch": 0.78125, "grad_norm": 1.5571203231811523, "learning_rate": 7.450608454068415e-05, "loss": 0.006761215627193451, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00678, "step": 200, "tokens/total": 6481824, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 91355} +{"epoch": 0.78515625, "grad_norm": 0.20197470486164093, "learning_rate": 7.424407941704987e-05, "loss": 0.005734847858548164, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00575, "step": 201, "tokens/total": 6515896, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 91856} +{"epoch": 0.7890625, "grad_norm": 0.4711218774318695, "learning_rate": 7.398127346871986e-05, "loss": 0.01877179741859436, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.01895, "step": 202, "tokens/total": 6548784, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 92311} +{"epoch": 0.79296875, "grad_norm": 0.15755809843540192, "learning_rate": 7.371767763212238e-05, "loss": 0.0019135441398248076, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00192, "step": 203, "tokens/total": 6580256, "tokens/train_per_sec_per_gpu": 10.18, "tokens/trainable": 92764} +{"epoch": 0.796875, "grad_norm": 0.0969487726688385, "learning_rate": 7.345330287655617e-05, "loss": 0.0014188297791406512, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00142, "step": 204, "tokens/total": 6613976, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 93227} +{"epoch": 0.80078125, "grad_norm": 0.26125404238700867, "learning_rate": 7.31881602037339e-05, "loss": 0.004926424939185381, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.00494, "step": 205, "tokens/total": 6644552, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 93675} +{"epoch": 0.8046875, "grad_norm": 0.0869775041937828, "learning_rate": 7.29222606473245e-05, "loss": 0.0010632923804223537, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00106, "step": 206, "tokens/total": 6675056, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 94120} +{"epoch": 0.80859375, "grad_norm": 0.09746749699115753, "learning_rate": 7.265561527249383e-05, "loss": 0.0011699087917804718, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00117, "step": 207, "tokens/total": 6706624, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 94557} +{"epoch": 0.8125, "grad_norm": 0.30821800231933594, "learning_rate": 7.238823517544436e-05, "loss": 0.005142273847013712, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00516, "step": 208, "tokens/total": 6741008, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 95035} +{"epoch": 0.81640625, "grad_norm": 0.5179352760314941, "learning_rate": 7.212013148295333e-05, "loss": 0.013008003123104572, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.01309, "step": 209, "tokens/total": 6774872, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 95517} +{"epoch": 0.8203125, "grad_norm": 0.07351481914520264, "learning_rate": 7.185131535190975e-05, "loss": 0.0009268443682231009, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00093, "step": 210, "tokens/total": 6805816, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 96026} +{"epoch": 0.82421875, "grad_norm": 0.1403493881225586, "learning_rate": 7.158179796885005e-05, "loss": 0.001201441278681159, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.0012, "step": 211, "tokens/total": 6837592, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 96477} +{"epoch": 0.828125, "grad_norm": 0.03815926983952522, "learning_rate": 7.131159054949273e-05, "loss": 0.0003451064112596214, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00035, "step": 212, "tokens/total": 6874384, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 96951} +{"epoch": 0.83203125, "grad_norm": 0.04716499522328377, "learning_rate": 7.104070433827139e-05, "loss": 0.00028511887649074197, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00029, "step": 213, "tokens/total": 6905872, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 97350} +{"epoch": 0.8359375, "grad_norm": 0.021537527441978455, "learning_rate": 7.076915060786705e-05, "loss": 0.00020273905829526484, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.0002, "step": 214, "tokens/total": 6937984, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 97792} +{"epoch": 0.83984375, "grad_norm": 0.20436730980873108, "learning_rate": 7.049694065873882e-05, "loss": 0.003042384050786495, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00305, "step": 215, "tokens/total": 6970704, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 98257} +{"epoch": 0.84375, "grad_norm": 0.17241553962230682, "learning_rate": 7.022408581865382e-05, "loss": 0.002273945603519678, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00228, "step": 216, "tokens/total": 7004616, "tokens/train_per_sec_per_gpu": 7.24, "tokens/trainable": 98731} +{"epoch": 0.84765625, "grad_norm": 0.004443285521119833, "learning_rate": 6.99505974422157e-05, "loss": 4.893411824014038e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00005, "step": 217, "tokens/total": 7036880, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 99212} +{"epoch": 0.8515625, "grad_norm": 0.02444218471646309, "learning_rate": 6.967648691039213e-05, "loss": 0.00014400588406715542, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00014, "step": 218, "tokens/total": 7068208, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 99654} +{"epoch": 0.85546875, "grad_norm": 0.5227293968200684, "learning_rate": 6.940176563004123e-05, "loss": 0.006770425476133823, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00679, "step": 219, "tokens/total": 7098112, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 100086} +{"epoch": 0.859375, "grad_norm": 0.05742228031158447, "learning_rate": 6.912644503343682e-05, "loss": 0.000598678772803396, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.0006, "step": 220, "tokens/total": 7132432, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 100524} +{"epoch": 0.86328125, "grad_norm": 0.01617586240172386, "learning_rate": 6.885053657779273e-05, "loss": 7.856288721086457e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.00008, "step": 221, "tokens/total": 7164016, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 100996} +{"epoch": 0.8671875, "grad_norm": 0.2644338607788086, "learning_rate": 6.857405174478604e-05, "loss": 0.0019114100141450763, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00191, "step": 222, "tokens/total": 7198344, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 101449} +{"epoch": 0.87109375, "grad_norm": 0.250561386346817, "learning_rate": 6.82970020400792e-05, "loss": 0.0024965296033769846, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.0025, "step": 223, "tokens/total": 7232192, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 101905} +{"epoch": 0.875, "grad_norm": 1.257630705833435, "learning_rate": 6.801939899284132e-05, "loss": 0.0146897928789258, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.0148, "step": 224, "tokens/total": 7266984, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 102411} +{"epoch": 0.87890625, "grad_norm": 0.25671079754829407, "learning_rate": 6.774125415526827e-05, "loss": 0.0025787935592234135, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00258, "step": 225, "tokens/total": 7299512, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 102882} +{"epoch": 0.8828125, "grad_norm": 0.018991945311427116, "learning_rate": 6.746257910210214e-05, "loss": 0.00010066870891023427, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.0001, "step": 226, "tokens/total": 7331120, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 103332} +{"epoch": 0.88671875, "grad_norm": 1.104308009147644, "learning_rate": 6.718338543014937e-05, "loss": 0.03588910400867462, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.03654, "step": 227, "tokens/total": 7360504, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 103790} +{"epoch": 0.890625, "grad_norm": 0.12279514968395233, "learning_rate": 6.69036847577983e-05, "loss": 0.0005490678013302386, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00055, "step": 228, "tokens/total": 7390760, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 104246} +{"epoch": 0.89453125, "grad_norm": 0.4412979781627655, "learning_rate": 6.662348872453553e-05, "loss": 0.009236601181328297, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00928, "step": 229, "tokens/total": 7423352, "tokens/train_per_sec_per_gpu": 10.59, "tokens/trainable": 104707} +{"epoch": 0.8984375, "grad_norm": 0.0030609022360295057, "learning_rate": 6.63428089904618e-05, "loss": 5.169376527192071e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00005, "step": 230, "tokens/total": 7457152, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 105167} +{"epoch": 0.90234375, "grad_norm": 0.010641081258654594, "learning_rate": 6.60616572358065e-05, "loss": 9.641245560487732e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.0001, "step": 231, "tokens/total": 7488648, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 105576} +{"epoch": 0.90625, "grad_norm": 0.006545208394527435, "learning_rate": 6.578004516044172e-05, "loss": 0.0001394421560689807, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00014, "step": 232, "tokens/total": 7522160, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 106043} +{"epoch": 0.91015625, "grad_norm": 0.37636956572532654, "learning_rate": 6.549798448339548e-05, "loss": 0.007614405360072851, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00764, "step": 233, "tokens/total": 7557336, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 106506} +{"epoch": 0.9140625, "grad_norm": 0.039182063192129135, "learning_rate": 6.521548694236384e-05, "loss": 0.0004945601103827357, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00049, "step": 234, "tokens/total": 7590432, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 106951} +{"epoch": 0.91796875, "grad_norm": 0.08892034739255905, "learning_rate": 6.493256429322259e-05, "loss": 0.001035888446494937, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00104, "step": 235, "tokens/total": 7623240, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 107382} +{"epoch": 0.921875, "grad_norm": 0.16875486075878143, "learning_rate": 6.464922830953799e-05, "loss": 0.0030380140524357557, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00304, "step": 236, "tokens/total": 7657968, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 107814} +{"epoch": 0.92578125, "grad_norm": 0.29559338092803955, "learning_rate": 6.436549078207688e-05, "loss": 0.009309997782111168, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00935, "step": 237, "tokens/total": 7689320, "tokens/train_per_sec_per_gpu": 10.22, "tokens/trainable": 108274} +{"epoch": 0.9296875, "grad_norm": 0.19524158537387848, "learning_rate": 6.408136351831592e-05, "loss": 0.003272023517638445, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00328, "step": 238, "tokens/total": 7721568, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 108714} +{"epoch": 0.93359375, "grad_norm": 0.03994755074381828, "learning_rate": 6.379685834195036e-05, "loss": 0.0006188689148984849, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00062, "step": 239, "tokens/total": 7756464, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 109220} +{"epoch": 0.9375, "grad_norm": 0.10749903321266174, "learning_rate": 6.351198709240186e-05, "loss": 0.0019054298754781485, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00191, "step": 240, "tokens/total": 7791616, "tokens/train_per_sec_per_gpu": 7.08, "tokens/trainable": 109672} +{"epoch": 0.94140625, "grad_norm": 0.30611473321914673, "learning_rate": 6.32267616243259e-05, "loss": 0.0031302799470722675, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00314, "step": 241, "tokens/total": 7823520, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 110135} +{"epoch": 0.9453125, "grad_norm": 0.07359737157821655, "learning_rate": 6.294119380711849e-05, "loss": 0.0013306443579494953, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00133, "step": 242, "tokens/total": 7854424, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 110563} +{"epoch": 0.94921875, "grad_norm": 0.5179994702339172, "learning_rate": 6.265529552442209e-05, "loss": 0.013286378234624863, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01338, "step": 243, "tokens/total": 7887160, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 111049} +{"epoch": 0.953125, "grad_norm": 0.0813293606042862, "learning_rate": 6.236907867363127e-05, "loss": 0.0010941002983599901, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00109, "step": 244, "tokens/total": 7918736, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 111495} +{"epoch": 0.95703125, "grad_norm": 0.26606619358062744, "learning_rate": 6.208255516539749e-05, "loss": 0.004293564707040787, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.0043, "step": 245, "tokens/total": 7949448, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 111968} +{"epoch": 0.9609375, "grad_norm": 0.023868583142757416, "learning_rate": 6.179573692313344e-05, "loss": 0.00021564256167039275, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00022, "step": 246, "tokens/total": 7984680, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 112416} +{"epoch": 0.96484375, "grad_norm": 0.027257531881332397, "learning_rate": 6.150863588251694e-05, "loss": 0.0003010330256074667, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.0003, "step": 247, "tokens/total": 8019816, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 112882} +{"epoch": 0.96875, "grad_norm": 0.3005468249320984, "learning_rate": 6.122126399099419e-05, "loss": 0.0041315751150250435, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00414, "step": 248, "tokens/total": 8056272, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 113390} +{"epoch": 0.97265625, "grad_norm": 0.2702334225177765, "learning_rate": 6.0933633207282615e-05, "loss": 0.005151937250047922, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00517, "step": 249, "tokens/total": 8090224, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 113904} +{"epoch": 0.9765625, "grad_norm": 0.03849097713828087, "learning_rate": 6.064575550087316e-05, "loss": 0.0006132201524451375, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00061, "step": 250, "tokens/total": 8124528, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 114351} +{"epoch": 0.98046875, "grad_norm": 0.06153414398431778, "learning_rate": 6.0357642851532245e-05, "loss": 0.0008633941179141402, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00086, "step": 251, "tokens/total": 8159688, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 114842} +{"epoch": 0.984375, "grad_norm": 0.06319469213485718, "learning_rate": 6.0069307248803294e-05, "loss": 0.0009957978036254644, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.001, "step": 252, "tokens/total": 8191800, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 115263} +{"epoch": 0.98828125, "grad_norm": 0.11376607418060303, "learning_rate": 5.9780760691507635e-05, "loss": 0.0018974815029650927, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.0019, "step": 253, "tokens/total": 8221752, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 115703} +{"epoch": 0.9921875, "grad_norm": 0.45245060324668884, "learning_rate": 5.9492015187245334e-05, "loss": 0.005052435677498579, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00507, "step": 254, "tokens/total": 8255648, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 116157} +{"epoch": 0.99609375, "grad_norm": 0.3557461202144623, "learning_rate": 5.920308275189541e-05, "loss": 0.004735275637358427, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00475, "step": 255, "tokens/total": 8286664, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 116567} +{"epoch": 1.0, "grad_norm": 0.08391796052455902, "learning_rate": 5.8913975409115874e-05, "loss": 0.000857892504427582, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00086, "step": 256, "tokens/total": 8318368, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 117030} +{"epoch": 1.00390625, "grad_norm": 0.06848722696304321, "learning_rate": 5.8624705189843395e-05, "loss": 0.0004784769844263792, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00048, "step": 257, "tokens/total": 8353336, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 117517} +{"epoch": 1.0078125, "grad_norm": 0.10039420425891876, "learning_rate": 5.833528413179249e-05, "loss": 0.0007591432658955455, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00076, "step": 258, "tokens/total": 8389112, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 118015} +{"epoch": 1.01171875, "grad_norm": 0.10225911438465118, "learning_rate": 5.80457242789548e-05, "loss": 0.0014312310377135873, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00143, "step": 259, "tokens/total": 8426000, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 118520} +{"epoch": 1.015625, "grad_norm": 0.06724807620048523, "learning_rate": 5.77560376810977e-05, "loss": 0.0006564376526512206, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.00066, "step": 260, "tokens/total": 8458568, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 118992} +{"epoch": 1.01953125, "grad_norm": 0.14450448751449585, "learning_rate": 5.7466236393263005e-05, "loss": 0.0010170812020078301, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00102, "step": 261, "tokens/total": 8490840, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 119438} +{"epoch": 1.0234375, "grad_norm": 0.007539001759141684, "learning_rate": 5.717633247526522e-05, "loss": 0.0001331511011812836, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00013, "step": 262, "tokens/total": 8521792, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 119881} +{"epoch": 1.02734375, "grad_norm": 0.013052014634013176, "learning_rate": 5.688633799118971e-05, "loss": 0.00019536714535206556, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.0002, "step": 263, "tokens/total": 8551336, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 120285} +{"epoch": 1.03125, "grad_norm": 0.21525532007217407, "learning_rate": 5.659626500889066e-05, "loss": 0.0024348306469619274, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00244, "step": 264, "tokens/total": 8579872, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 120755} +{"epoch": 1.03515625, "grad_norm": 1.075710654258728, "learning_rate": 5.6306125599488905e-05, "loss": 0.015528545714914799, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.01565, "step": 265, "tokens/total": 8608920, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 121194} +{"epoch": 1.0390625, "grad_norm": 0.02005176804959774, "learning_rate": 5.601593183686955e-05, "loss": 0.00019812247774098068, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.0002, "step": 266, "tokens/total": 8643600, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 121670} +{"epoch": 1.04296875, "grad_norm": 0.8737048506736755, "learning_rate": 5.572569579717961e-05, "loss": 0.0036575142294168472, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00366, "step": 267, "tokens/total": 8676032, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 122142} +{"epoch": 1.046875, "grad_norm": 0.43123704195022583, "learning_rate": 5.543542955832538e-05, "loss": 0.0023210691288113594, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00232, "step": 268, "tokens/total": 8713240, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 122585} +{"epoch": 1.05078125, "grad_norm": 0.2298411875963211, "learning_rate": 5.514514519946986e-05, "loss": 0.0012919785222038627, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00129, "step": 269, "tokens/total": 8751664, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 123091} +{"epoch": 1.0546875, "grad_norm": 0.06770119071006775, "learning_rate": 5.485485480053015e-05, "loss": 0.0004930758150294423, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00049, "step": 270, "tokens/total": 8786112, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 123505} +{"epoch": 1.05859375, "grad_norm": 0.004486800637096167, "learning_rate": 5.4564570441674645e-05, "loss": 4.356203862698749e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00004, "step": 271, "tokens/total": 8816920, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 123953} +{"epoch": 1.0625, "grad_norm": 0.15873421728610992, "learning_rate": 5.42743042028204e-05, "loss": 0.0004152297042310238, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00042, "step": 272, "tokens/total": 8851408, "tokens/train_per_sec_per_gpu": 6.87, "tokens/trainable": 124400} +{"epoch": 1.06640625, "grad_norm": 0.15402550995349884, "learning_rate": 5.3984068163130464e-05, "loss": 0.0006767035229131579, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00068, "step": 273, "tokens/total": 8883768, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 124870} +{"epoch": 1.0703125, "grad_norm": 0.4435349404811859, "learning_rate": 5.369387440051111e-05, "loss": 0.013293030671775341, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.01338, "step": 274, "tokens/total": 8916200, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 125333} +{"epoch": 1.07421875, "grad_norm": 0.005021009128540754, "learning_rate": 5.340373499110935e-05, "loss": 4.681749123847112e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00005, "step": 275, "tokens/total": 8949920, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 125834} +{"epoch": 1.078125, "grad_norm": 0.023695629090070724, "learning_rate": 5.3113662008810304e-05, "loss": 0.00013837762526236475, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00014, "step": 276, "tokens/total": 8986448, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 126316} +{"epoch": 1.08203125, "grad_norm": 0.09304496645927429, "learning_rate": 5.282366752473479e-05, "loss": 0.000309570983517915, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00031, "step": 277, "tokens/total": 9020600, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 126798} +{"epoch": 1.0859375, "grad_norm": 0.04669291898608208, "learning_rate": 5.2533763606737005e-05, "loss": 0.0004932149313390255, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00049, "step": 278, "tokens/total": 9056040, "tokens/train_per_sec_per_gpu": 6.92, "tokens/trainable": 127223} +{"epoch": 1.08984375, "grad_norm": 0.025443006306886673, "learning_rate": 5.224396231890232e-05, "loss": 0.0002743343939073384, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00027, "step": 279, "tokens/total": 9087072, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 127672} +{"epoch": 1.09375, "grad_norm": 0.149648979306221, "learning_rate": 5.195427572104522e-05, "loss": 0.0016494421288371086, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00165, "step": 280, "tokens/total": 9118376, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 128116} +{"epoch": 1.09765625, "grad_norm": 0.16323158144950867, "learning_rate": 5.166471586820751e-05, "loss": 0.0009798876708373427, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00098, "step": 281, "tokens/total": 9152704, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 128589} +{"epoch": 1.1015625, "grad_norm": 1.5263515710830688, "learning_rate": 5.1375294810156615e-05, "loss": 0.007325394079089165, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00735, "step": 282, "tokens/total": 9184448, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 129036} +{"epoch": 1.10546875, "grad_norm": 0.22727566957473755, "learning_rate": 5.1086024590884144e-05, "loss": 0.0012779362732544541, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00128, "step": 283, "tokens/total": 9217392, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 129485} +{"epoch": 1.109375, "grad_norm": 0.06259222328662872, "learning_rate": 5.079691724810461e-05, "loss": 0.0008430395973846316, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00084, "step": 284, "tokens/total": 9252016, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 129920} +{"epoch": 1.11328125, "grad_norm": 0.1577448546886444, "learning_rate": 5.0507984812754684e-05, "loss": 0.0008352997247129679, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00084, "step": 285, "tokens/total": 9284008, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 130417} +{"epoch": 1.1171875, "grad_norm": 0.023832496255636215, "learning_rate": 5.021923930849237e-05, "loss": 0.00026267155772075057, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00026, "step": 286, "tokens/total": 9315984, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 130903} +{"epoch": 1.12109375, "grad_norm": 0.0035140349064022303, "learning_rate": 4.99306927511967e-05, "loss": 4.95451022288762e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00005, "step": 287, "tokens/total": 9347376, "tokens/train_per_sec_per_gpu": 10.49, "tokens/trainable": 131343} +{"epoch": 1.125, "grad_norm": 0.5373466610908508, "learning_rate": 4.964235714846775e-05, "loss": 0.008302640169858932, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00834, "step": 288, "tokens/total": 9377424, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 131763} +{"epoch": 1.12890625, "grad_norm": 0.1368928700685501, "learning_rate": 4.9354244499126866e-05, "loss": 0.000570565985981375, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00057, "step": 289, "tokens/total": 9408104, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 132229} +{"epoch": 1.1328125, "grad_norm": 0.14649516344070435, "learning_rate": 4.90663667927174e-05, "loss": 0.0012467763153836131, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00125, "step": 290, "tokens/total": 9439944, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 132678} +{"epoch": 1.13671875, "grad_norm": 0.0066320099867880344, "learning_rate": 4.877873600900581e-05, "loss": 8.296249143313617e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00008, "step": 291, "tokens/total": 9472104, "tokens/train_per_sec_per_gpu": 6.26, "tokens/trainable": 133136} +{"epoch": 1.140625, "grad_norm": 0.0486949160695076, "learning_rate": 4.849136411748306e-05, "loss": 0.00035660312278196216, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00036, "step": 292, "tokens/total": 9508336, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 133608} +{"epoch": 1.14453125, "grad_norm": 0.04367189109325409, "learning_rate": 4.8204263076866574e-05, "loss": 0.00029964983696117997, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.0003, "step": 293, "tokens/total": 9541552, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 134108} +{"epoch": 1.1484375, "grad_norm": 0.010170720517635345, "learning_rate": 4.791744483460251e-05, "loss": 8.963667642092332e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00009, "step": 294, "tokens/total": 9575072, "tokens/train_per_sec_per_gpu": 6.64, "tokens/trainable": 134541} +{"epoch": 1.15234375, "grad_norm": 0.0033758855424821377, "learning_rate": 4.7630921326368736e-05, "loss": 5.873064583283849e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00006, "step": 295, "tokens/total": 9610344, "tokens/train_per_sec_per_gpu": 6.46, "tokens/trainable": 134966} +{"epoch": 1.15625, "grad_norm": 0.06908180564641953, "learning_rate": 4.7344704475577916e-05, "loss": 0.00040676462231203914, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00041, "step": 296, "tokens/total": 9639728, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 135402} +{"epoch": 1.16015625, "grad_norm": 0.47179844975471497, "learning_rate": 4.705880619288153e-05, "loss": 0.009840627200901508, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00989, "step": 297, "tokens/total": 9671424, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 135804} +{"epoch": 1.1640625, "grad_norm": 0.009322447702288628, "learning_rate": 4.677323837567412e-05, "loss": 0.00011800527136074379, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00012, "step": 298, "tokens/total": 9700808, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 136231} +{"epoch": 1.16796875, "grad_norm": 0.23766867816448212, "learning_rate": 4.6488012907598146e-05, "loss": 0.0012947029899805784, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.0013, "step": 299, "tokens/total": 9732312, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 136705} +{"epoch": 1.171875, "grad_norm": 0.26715099811553955, "learning_rate": 4.620314165804964e-05, "loss": 0.0028176039922982454, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00282, "step": 300, "tokens/total": 9764624, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 137178} +{"epoch": 1.17578125, "grad_norm": 0.10420452803373337, "learning_rate": 4.591863648168407e-05, "loss": 0.0004736201371997595, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00047, "step": 301, "tokens/total": 9798368, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 137643} +{"epoch": 1.1796875, "grad_norm": 0.3019394278526306, "learning_rate": 4.5634509217923135e-05, "loss": 0.012794019654393196, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.01288, "step": 302, "tokens/total": 9830192, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 138078} +{"epoch": 1.18359375, "grad_norm": 1.1295149326324463, "learning_rate": 4.535077169046201e-05, "loss": 0.005826032254844904, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00584, "step": 303, "tokens/total": 9863112, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 138515} +{"epoch": 1.1875, "grad_norm": 0.09968335181474686, "learning_rate": 4.506743570677743e-05, "loss": 0.0011028244625777006, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.0011, "step": 304, "tokens/total": 9893192, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 138948} +{"epoch": 1.19140625, "grad_norm": 0.19552578032016754, "learning_rate": 4.478451305763618e-05, "loss": 0.001325338613241911, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00133, "step": 305, "tokens/total": 9925712, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 139408} +{"epoch": 1.1953125, "grad_norm": 0.0013826885260641575, "learning_rate": 4.450201551660454e-05, "loss": 3.378833571332507e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00003, "step": 306, "tokens/total": 9955736, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 139840} +{"epoch": 1.19921875, "grad_norm": 0.4080502986907959, "learning_rate": 4.4219954839558276e-05, "loss": 0.009989986196160316, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01004, "step": 307, "tokens/total": 9987160, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 140281} +{"epoch": 1.203125, "grad_norm": 0.06353618204593658, "learning_rate": 4.393834276419352e-05, "loss": 0.0005161280278116465, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00052, "step": 308, "tokens/total": 10017360, "tokens/train_per_sec_per_gpu": 11.21, "tokens/trainable": 140776} +{"epoch": 1.20703125, "grad_norm": 0.3528420329093933, "learning_rate": 4.36571910095382e-05, "loss": 0.0012274958426132798, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00123, "step": 309, "tokens/total": 10052056, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 141228} +{"epoch": 1.2109375, "grad_norm": 0.35880541801452637, "learning_rate": 4.337651127546448e-05, "loss": 0.010909413918852806, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01097, "step": 310, "tokens/total": 10085424, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 141679} +{"epoch": 1.21484375, "grad_norm": 0.32196393609046936, "learning_rate": 4.3096315242201736e-05, "loss": 0.007254816126078367, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00728, "step": 311, "tokens/total": 10116824, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 142122} +{"epoch": 1.21875, "grad_norm": 0.005861103534698486, "learning_rate": 4.2816614569850635e-05, "loss": 7.851072587072849e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00008, "step": 312, "tokens/total": 10150376, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 142561} +{"epoch": 1.22265625, "grad_norm": 0.04393322020769119, "learning_rate": 4.2537420897897864e-05, "loss": 0.0005298781907185912, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00053, "step": 313, "tokens/total": 10184432, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 143046} +{"epoch": 1.2265625, "grad_norm": 0.07531175762414932, "learning_rate": 4.225874584473174e-05, "loss": 0.0008362902444787323, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00084, "step": 314, "tokens/total": 10214848, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 143493} +{"epoch": 1.23046875, "grad_norm": 0.13271009922027588, "learning_rate": 4.19806010071587e-05, "loss": 0.0008132412331178784, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00081, "step": 315, "tokens/total": 10247952, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 143956} +{"epoch": 1.234375, "grad_norm": 0.17070825397968292, "learning_rate": 4.170299795992081e-05, "loss": 0.0011097498936578631, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00111, "step": 316, "tokens/total": 10280648, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 144411} +{"epoch": 1.23828125, "grad_norm": 0.017863988876342773, "learning_rate": 4.142594825521398e-05, "loss": 0.00017595727695152164, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00018, "step": 317, "tokens/total": 10317712, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 144892} +{"epoch": 1.2421875, "grad_norm": 0.02437025122344494, "learning_rate": 4.114946342220728e-05, "loss": 0.0002291285782121122, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00023, "step": 318, "tokens/total": 10351632, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 145339} +{"epoch": 1.24609375, "grad_norm": 0.019761845469474792, "learning_rate": 4.087355496656321e-05, "loss": 0.0002335557946935296, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00023, "step": 319, "tokens/total": 10383336, "tokens/train_per_sec_per_gpu": 10.23, "tokens/trainable": 145811} +{"epoch": 1.25, "grad_norm": 0.06066030263900757, "learning_rate": 4.05982343699588e-05, "loss": 0.0006820284761488438, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00068, "step": 320, "tokens/total": 10418016, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 146314} +{"epoch": 1.25390625, "grad_norm": 0.0899992287158966, "learning_rate": 4.0323513089607876e-05, "loss": 0.0010014078579843044, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.001, "step": 321, "tokens/total": 10451600, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 146781} +{"epoch": 1.2578125, "grad_norm": 0.08520933240652084, "learning_rate": 4.004940255778431e-05, "loss": 0.0009900431614369154, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00099, "step": 322, "tokens/total": 10487568, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 147225} +{"epoch": 1.26171875, "grad_norm": 0.09920624643564224, "learning_rate": 3.977591418134619e-05, "loss": 0.0009311063913628459, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00093, "step": 323, "tokens/total": 10520976, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 147673} +{"epoch": 1.265625, "grad_norm": 0.024602269753813744, "learning_rate": 3.95030593412612e-05, "loss": 0.00044725689804181457, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00045, "step": 324, "tokens/total": 10552776, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 148103} +{"epoch": 1.26953125, "grad_norm": 0.021535569801926613, "learning_rate": 3.923084939213296e-05, "loss": 0.00023680960293859243, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00024, "step": 325, "tokens/total": 10582576, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 148535} +{"epoch": 1.2734375, "grad_norm": 0.027933131903409958, "learning_rate": 3.895929566172861e-05, "loss": 0.0003803727449849248, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00038, "step": 326, "tokens/total": 10615600, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 148999} +{"epoch": 1.27734375, "grad_norm": 0.015768636018037796, "learning_rate": 3.868840945050728e-05, "loss": 0.00022829734371043742, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00023, "step": 327, "tokens/total": 10650688, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 149431} +{"epoch": 1.28125, "grad_norm": 0.024408306926488876, "learning_rate": 3.841820203114995e-05, "loss": 0.0002776627952698618, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00028, "step": 328, "tokens/total": 10681816, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 149886} +{"epoch": 1.28515625, "grad_norm": 0.050517771393060684, "learning_rate": 3.814868464809027e-05, "loss": 0.00014682284381706268, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00015, "step": 329, "tokens/total": 10711728, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 150288} +{"epoch": 1.2890625, "grad_norm": 0.06788556277751923, "learning_rate": 3.787986851704667e-05, "loss": 0.000541945337317884, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00054, "step": 330, "tokens/total": 10744952, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 150733} +{"epoch": 1.29296875, "grad_norm": 0.030831076204776764, "learning_rate": 3.7611764824555654e-05, "loss": 0.0003245876869186759, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00032, "step": 331, "tokens/total": 10776192, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 151207} +{"epoch": 1.296875, "grad_norm": 0.268972784280777, "learning_rate": 3.734438472750619e-05, "loss": 0.0014302373165264726, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00143, "step": 332, "tokens/total": 10813336, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 151694} +{"epoch": 1.30078125, "grad_norm": 0.004114050418138504, "learning_rate": 3.707773935267552e-05, "loss": 8.63000750541687e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00009, "step": 333, "tokens/total": 10847808, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 152188} +{"epoch": 1.3046875, "grad_norm": 0.003715542610734701, "learning_rate": 3.68118397962661e-05, "loss": 4.577574509312399e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00005, "step": 334, "tokens/total": 10879896, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 152596} +{"epoch": 1.30859375, "grad_norm": 0.7371000647544861, "learning_rate": 3.654669712344384e-05, "loss": 0.005083797499537468, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.0051, "step": 335, "tokens/total": 10908976, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 153052} +{"epoch": 1.3125, "grad_norm": 0.15074710547924042, "learning_rate": 3.628232236787763e-05, "loss": 0.0011207833886146545, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00112, "step": 336, "tokens/total": 10943152, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 153491} +{"epoch": 1.31640625, "grad_norm": 0.014162360690534115, "learning_rate": 3.6018726531280144e-05, "loss": 0.0002690521941985935, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00027, "step": 337, "tokens/total": 10978184, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 153983} +{"epoch": 1.3203125, "grad_norm": 0.07749667018651962, "learning_rate": 3.575592058295017e-05, "loss": 0.0007755120168440044, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00078, "step": 338, "tokens/total": 11010056, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 154464} +{"epoch": 1.32421875, "grad_norm": 0.006739678326994181, "learning_rate": 3.549391545931585e-05, "loss": 8.557504770578817e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00009, "step": 339, "tokens/total": 11040968, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 154924} +{"epoch": 1.328125, "grad_norm": 0.0054307240061461926, "learning_rate": 3.5232722063479914e-05, "loss": 8.230004459619522e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00008, "step": 340, "tokens/total": 11073424, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 155373} +{"epoch": 1.33203125, "grad_norm": 0.2544925808906555, "learning_rate": 3.49723512647657e-05, "loss": 0.0013272598152980208, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00133, "step": 341, "tokens/total": 11109224, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 155873} +{"epoch": 1.3359375, "grad_norm": 0.002216503955423832, "learning_rate": 3.471281389826491e-05, "loss": 3.910969462594949e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00004, "step": 342, "tokens/total": 11139856, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 156278} +{"epoch": 1.33984375, "grad_norm": 0.005261674057692289, "learning_rate": 3.4454120764386764e-05, "loss": 6.769588071620092e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00007, "step": 343, "tokens/total": 11175224, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 156733} +{"epoch": 1.34375, "grad_norm": 0.007012222893536091, "learning_rate": 3.4196282628408526e-05, "loss": 3.788423418882303e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00004, "step": 344, "tokens/total": 11207112, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 157203} +{"epoch": 1.34765625, "grad_norm": 0.008538120426237583, "learning_rate": 3.3939310220027456e-05, "loss": 8.371458534384146e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00008, "step": 345, "tokens/total": 11240432, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 157707} +{"epoch": 1.3515625, "grad_norm": 0.0016427800292149186, "learning_rate": 3.3683214232914404e-05, "loss": 2.3635642719455063e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00002, "step": 346, "tokens/total": 11274584, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 158180} +{"epoch": 1.35546875, "grad_norm": 0.010871698148548603, "learning_rate": 3.342800532426873e-05, "loss": 0.0001052787629305385, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00011, "step": 347, "tokens/total": 11307376, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 158646} +{"epoch": 1.359375, "grad_norm": 0.08639048784971237, "learning_rate": 3.317369411437484e-05, "loss": 0.000842646521050483, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00084, "step": 348, "tokens/total": 11340640, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 159115} +{"epoch": 1.36328125, "grad_norm": 0.020214615389704704, "learning_rate": 3.292029118616024e-05, "loss": 5.9823716583196074e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00006, "step": 349, "tokens/total": 11371632, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 159538} +{"epoch": 1.3671875, "grad_norm": 0.1905047446489334, "learning_rate": 3.266780708475511e-05, "loss": 0.001997431041672826, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.002, "step": 350, "tokens/total": 11404248, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 160016} +{"epoch": 1.37109375, "grad_norm": 0.01228930801153183, "learning_rate": 3.241625231705354e-05, "loss": 0.00011861516395583749, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00012, "step": 351, "tokens/total": 11439552, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 160475} +{"epoch": 1.375, "grad_norm": 0.0036533772945404053, "learning_rate": 3.216563735127618e-05, "loss": 4.699920100392774e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00005, "step": 352, "tokens/total": 11470944, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 160952} +{"epoch": 1.37890625, "grad_norm": 0.2779008448123932, "learning_rate": 3.191597261653475e-05, "loss": 0.0020965775474905968, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.0021, "step": 353, "tokens/total": 11506304, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 161420} +{"epoch": 1.3828125, "grad_norm": 0.23505249619483948, "learning_rate": 3.166726850239794e-05, "loss": 0.004818720743060112, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00483, "step": 354, "tokens/total": 11535424, "tokens/train_per_sec_per_gpu": 11.07, "tokens/trainable": 161868} +{"epoch": 1.38671875, "grad_norm": 0.008163323625922203, "learning_rate": 3.141953535845912e-05, "loss": 5.70355769013986e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00006, "step": 355, "tokens/total": 11565176, "tokens/train_per_sec_per_gpu": 10.34, "tokens/trainable": 162318} +{"epoch": 1.390625, "grad_norm": 0.03407634422183037, "learning_rate": 3.11727834939056e-05, "loss": 0.00016564365068916231, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00017, "step": 356, "tokens/total": 11595528, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 162789} +{"epoch": 1.39453125, "grad_norm": 0.31747815012931824, "learning_rate": 3.092702317708967e-05, "loss": 0.003229189198464155, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00323, "step": 357, "tokens/total": 11627248, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 163254} +{"epoch": 1.3984375, "grad_norm": 0.002052880823612213, "learning_rate": 3.0682264635101276e-05, "loss": 3.522224506014027e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00004, "step": 358, "tokens/total": 11659440, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 163715} +{"epoch": 1.40234375, "grad_norm": 0.010986451990902424, "learning_rate": 3.0438518053342407e-05, "loss": 3.128254320472479e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00003, "step": 359, "tokens/total": 11693288, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 164197} +{"epoch": 1.40625, "grad_norm": 0.0007573979091830552, "learning_rate": 3.0195793575103266e-05, "loss": 2.0406925614224747e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00002, "step": 360, "tokens/total": 11730888, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 164661} +{"epoch": 1.41015625, "grad_norm": 0.8234980702400208, "learning_rate": 2.9954101301140146e-05, "loss": 0.002608831040561199, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00261, "step": 361, "tokens/total": 11765656, "tokens/train_per_sec_per_gpu": 6.95, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.009650086052715778, "learning_rate": 2.9713451289255123e-05, "loss": 0.00011642770550679415, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00012, "step": 362, "tokens/total": 11793456, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 165552} +{"epoch": 1.41796875, "grad_norm": 0.006727229803800583, "learning_rate": 2.9473853553877484e-05, "loss": 5.989136116113514e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00006, "step": 363, "tokens/total": 11826280, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 166026} +{"epoch": 1.421875, "grad_norm": 0.002168006030842662, "learning_rate": 2.9235318065647e-05, "loss": 3.0288458219729364e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00003, "step": 364, "tokens/total": 11863552, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 166486} +{"epoch": 1.42578125, "grad_norm": 0.00046073735575191677, "learning_rate": 2.8997854750998964e-05, "loss": 1.2031857295369264e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00001, "step": 365, "tokens/total": 11898616, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 166959} +{"epoch": 1.4296875, "grad_norm": 0.002379166427999735, "learning_rate": 2.8761473491751258e-05, "loss": 3.415414539631456e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00003, "step": 366, "tokens/total": 11930728, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 167394} +{"epoch": 1.43359375, "grad_norm": 0.24672137200832367, "learning_rate": 2.8526184124692883e-05, "loss": 0.002159061376005411, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00216, "step": 367, "tokens/total": 11964976, "tokens/train_per_sec_per_gpu": 6.44, "tokens/trainable": 167832} +{"epoch": 1.4375, "grad_norm": 0.01231852825731039, "learning_rate": 2.829199644117484e-05, "loss": 0.0001443807122996077, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00014, "step": 368, "tokens/total": 11996696, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 168242} +{"epoch": 1.44140625, "grad_norm": 0.005252575967460871, "learning_rate": 2.8058920186702553e-05, "loss": 6.0669073718599975e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00006, "step": 369, "tokens/total": 12033432, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 168715} +{"epoch": 1.4453125, "grad_norm": 0.0032536392100155354, "learning_rate": 2.782696506053033e-05, "loss": 2.866219074348919e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00003, "step": 370, "tokens/total": 12067616, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 169147} +{"epoch": 1.44921875, "grad_norm": 0.0014939492102712393, "learning_rate": 2.7596140715257824e-05, "loss": 2.3133237846195698e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00002, "step": 371, "tokens/total": 12099008, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 169628} +{"epoch": 1.453125, "grad_norm": 0.010693025775253773, "learning_rate": 2.7366456756428184e-05, "loss": 9.647808474255726e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.0001, "step": 372, "tokens/total": 12131304, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 170085} +{"epoch": 1.45703125, "grad_norm": 0.21101976931095123, "learning_rate": 2.7137922742128486e-05, "loss": 0.002234336221590638, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00224, "step": 373, "tokens/total": 12164104, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 170584} +{"epoch": 1.4609375, "grad_norm": 0.23125997185707092, "learning_rate": 2.691054818259188e-05, "loss": 0.0034044156782329082, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00341, "step": 374, "tokens/total": 12197776, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 171058} +{"epoch": 1.46484375, "grad_norm": 0.24554257094860077, "learning_rate": 2.6684342539801933e-05, "loss": 0.0016843497287482023, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00169, "step": 375, "tokens/total": 12228064, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 171518} +{"epoch": 1.46875, "grad_norm": 0.05177733302116394, "learning_rate": 2.645931522709877e-05, "loss": 0.0003142921777907759, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00031, "step": 376, "tokens/total": 12261688, "tokens/train_per_sec_per_gpu": 10.84, "tokens/trainable": 172007} +{"epoch": 1.47265625, "grad_norm": 0.24261607229709625, "learning_rate": 2.6235475608787365e-05, "loss": 0.0004048843984492123, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.0004, "step": 377, "tokens/total": 12294240, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 172468} +{"epoch": 1.4765625, "grad_norm": 0.05658433958888054, "learning_rate": 2.6012832999747916e-05, "loss": 0.0004922595107927918, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00049, "step": 378, "tokens/total": 12324528, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 172923} +{"epoch": 1.48046875, "grad_norm": 0.5157670974731445, "learning_rate": 2.579139666504821e-05, "loss": 0.017318276688456535, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01747, "step": 379, "tokens/total": 12360288, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 173370} +{"epoch": 1.484375, "grad_norm": 0.0013472478603944182, "learning_rate": 2.557117581955798e-05, "loss": 1.8814009308698587e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00002, "step": 380, "tokens/total": 12391216, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 173837} +{"epoch": 1.48828125, "grad_norm": 0.006713980808854103, "learning_rate": 2.5352179627565532e-05, "loss": 7.887244282756001e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00008, "step": 381, "tokens/total": 12423624, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 174294} +{"epoch": 1.4921875, "grad_norm": 0.0019453878048807383, "learning_rate": 2.5134417202396277e-05, "loss": 2.1972020476823673e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00002, "step": 382, "tokens/total": 12456032, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 174701} +{"epoch": 1.49609375, "grad_norm": 0.1534063071012497, "learning_rate": 2.491789760603361e-05, "loss": 0.0022172038443386555, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00222, "step": 383, "tokens/total": 12489336, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 175131} +{"epoch": 1.5, "grad_norm": 0.4008817672729492, "learning_rate": 2.4702629848741764e-05, "loss": 0.0030314770992845297, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00304, "step": 384, "tokens/total": 12521944, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 175586} +{"epoch": 1.50390625, "grad_norm": 0.5147829055786133, "learning_rate": 2.4488622888690785e-05, "loss": 0.0005255639553070068, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00053, "step": 385, "tokens/total": 12554024, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 176026} +{"epoch": 1.5078125, "grad_norm": 0.0029386021196842194, "learning_rate": 2.427588563158384e-05, "loss": 3.337460657348856e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00003, "step": 386, "tokens/total": 12587104, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 176512} +{"epoch": 1.51171875, "grad_norm": 0.01640508510172367, "learning_rate": 2.406442693028651e-05, "loss": 0.00015150298713706434, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00015, "step": 387, "tokens/total": 12617456, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 176943} +{"epoch": 1.515625, "grad_norm": 0.010522016324102879, "learning_rate": 2.3854255584458547e-05, "loss": 0.00012601452181115746, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00013, "step": 388, "tokens/total": 12652696, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 177370} +{"epoch": 1.51953125, "grad_norm": 0.02548154443502426, "learning_rate": 2.3645380340187508e-05, "loss": 3.855057366308756e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00004, "step": 389, "tokens/total": 12685792, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 177851} +{"epoch": 1.5234375, "grad_norm": 0.02331264317035675, "learning_rate": 2.3437809889624914e-05, "loss": 0.00010162388207390904, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.0001, "step": 390, "tokens/total": 12716120, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 178310} +{"epoch": 1.52734375, "grad_norm": 0.0040094670839607716, "learning_rate": 2.3231552870624487e-05, "loss": 3.85335624741856e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00004, "step": 391, "tokens/total": 12746216, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 178736} +{"epoch": 1.53125, "grad_norm": 0.0022113770246505737, "learning_rate": 2.3026617866382657e-05, "loss": 3.7819103454239666e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00004, "step": 392, "tokens/total": 12778592, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 179233} +{"epoch": 1.53515625, "grad_norm": 0.00719884829595685, "learning_rate": 2.2823013405081507e-05, "loss": 3.5635908716358244e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00004, "step": 393, "tokens/total": 12812008, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 179730} +{"epoch": 1.5390625, "grad_norm": 0.02143479697406292, "learning_rate": 2.2620747959533722e-05, "loss": 0.0001419113395968452, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00014, "step": 394, "tokens/total": 12844264, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 180227} +{"epoch": 1.54296875, "grad_norm": 0.014722498133778572, "learning_rate": 2.2419829946830123e-05, "loss": 0.00013255795056466013, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00013, "step": 395, "tokens/total": 12878328, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 180687} +{"epoch": 1.546875, "grad_norm": 0.024943960830569267, "learning_rate": 2.2220267727989325e-05, "loss": 0.0002544362796470523, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00025, "step": 396, "tokens/total": 12907912, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 181141} +{"epoch": 1.55078125, "grad_norm": 0.06389918178319931, "learning_rate": 2.202206960760984e-05, "loss": 0.001025127130560577, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00103, "step": 397, "tokens/total": 12941968, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 181648} +{"epoch": 1.5546875, "grad_norm": 0.06337692588567734, "learning_rate": 2.182524383352446e-05, "loss": 0.0005605088081210852, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00056, "step": 398, "tokens/total": 12975240, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 182109} +{"epoch": 1.55859375, "grad_norm": 0.02454049326479435, "learning_rate": 2.1629798596457056e-05, "loss": 0.00026435096515342593, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00026, "step": 399, "tokens/total": 13006912, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 182559} +{"epoch": 1.5625, "grad_norm": 0.1955619603395462, "learning_rate": 2.1435742029681725e-05, "loss": 0.001244257902726531, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00125, "step": 400, "tokens/total": 13037640, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 182985} +{"epoch": 1.56640625, "grad_norm": 0.0009386029560118914, "learning_rate": 2.124308220868431e-05, "loss": 2.2133823222247884e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00002, "step": 401, "tokens/total": 13070608, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 183440} +{"epoch": 1.5703125, "grad_norm": 0.055716078728437424, "learning_rate": 2.105182715082638e-05, "loss": 0.0005007701693102717, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.0005, "step": 402, "tokens/total": 13101784, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 183917} +{"epoch": 1.57421875, "grad_norm": 0.1634742021560669, "learning_rate": 2.0861984815011552e-05, "loss": 0.001917041838169098, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00192, "step": 403, "tokens/total": 13133528, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 184348} +{"epoch": 1.578125, "grad_norm": 0.0015686770202592015, "learning_rate": 2.0673563101354323e-05, "loss": 3.471504169283435e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00003, "step": 404, "tokens/total": 13167088, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 184786} +{"epoch": 1.58203125, "grad_norm": 0.5377359390258789, "learning_rate": 2.0486569850851317e-05, "loss": 0.005656123626977205, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00567, "step": 405, "tokens/total": 13200392, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 185249} +{"epoch": 1.5859375, "grad_norm": 0.033076271414756775, "learning_rate": 2.0301012845054956e-05, "loss": 0.0004349870141595602, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00044, "step": 406, "tokens/total": 13235488, "tokens/train_per_sec_per_gpu": 6.92, "tokens/trainable": 185680} +{"epoch": 1.58984375, "grad_norm": 0.1983674317598343, "learning_rate": 2.011689980574966e-05, "loss": 0.0017061835387721658, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00171, "step": 407, "tokens/total": 13269416, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 186100} +{"epoch": 1.59375, "grad_norm": 0.09132345020771027, "learning_rate": 1.993423839463052e-05, "loss": 0.0008098019752651453, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00081, "step": 408, "tokens/total": 13301360, "tokens/train_per_sec_per_gpu": 10.48, "tokens/trainable": 186565} +{"epoch": 1.59765625, "grad_norm": 0.0012942986795678735, "learning_rate": 1.975303621298445e-05, "loss": 2.8469368771766312e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00003, "step": 409, "tokens/total": 13336392, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 187017} +{"epoch": 1.6015625, "grad_norm": 0.006498710252344608, "learning_rate": 1.957330080137385e-05, "loss": 5.250300819170661e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00005, "step": 410, "tokens/total": 13366072, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 187468} +{"epoch": 1.60546875, "grad_norm": 0.011357925832271576, "learning_rate": 1.9395039639322864e-05, "loss": 0.00013176568609196693, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00013, "step": 411, "tokens/total": 13400832, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 187961} +{"epoch": 1.609375, "grad_norm": 0.004673577379435301, "learning_rate": 1.9218260145006073e-05, "loss": 2.4907192710088566e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00002, "step": 412, "tokens/total": 13434920, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 188447} +{"epoch": 1.61328125, "grad_norm": 0.0021016686223447323, "learning_rate": 1.904296967493982e-05, "loss": 3.7083475035615265e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00004, "step": 413, "tokens/total": 13466512, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 188903} +{"epoch": 1.6171875, "grad_norm": 0.11608637124300003, "learning_rate": 1.8869175523676064e-05, "loss": 0.0013833476696163416, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00138, "step": 414, "tokens/total": 13500352, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 189398} +{"epoch": 1.62109375, "grad_norm": 0.014089812524616718, "learning_rate": 1.869688492349885e-05, "loss": 0.00011970988271059468, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00012, "step": 415, "tokens/total": 13532448, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 189850} +{"epoch": 1.625, "grad_norm": 0.24766604602336884, "learning_rate": 1.85261050441233e-05, "loss": 0.0007721613510511816, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00077, "step": 416, "tokens/total": 13565648, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 190306} +{"epoch": 1.62890625, "grad_norm": 0.16018737852573395, "learning_rate": 1.8356842992397304e-05, "loss": 0.0014534394722431898, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00145, "step": 417, "tokens/total": 13597600, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 190746} +{"epoch": 1.6328125, "grad_norm": 0.011253838427364826, "learning_rate": 1.8189105812005714e-05, "loss": 9.740314271766692e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.0001, "step": 418, "tokens/total": 13630456, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 191156} +{"epoch": 1.63671875, "grad_norm": 0.34943434596061707, "learning_rate": 1.802290048317732e-05, "loss": 0.002336052944883704, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00234, "step": 419, "tokens/total": 13660896, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 191573} +{"epoch": 1.640625, "grad_norm": 0.005575946997851133, "learning_rate": 1.785823392239424e-05, "loss": 4.022185748908669e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00004, "step": 420, "tokens/total": 13693176, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 192014} +{"epoch": 1.64453125, "grad_norm": 0.03772759065032005, "learning_rate": 1.7695112982104225e-05, "loss": 0.00036947158514522016, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00037, "step": 421, "tokens/total": 13725400, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 192453} +{"epoch": 1.6484375, "grad_norm": 0.08306970447301865, "learning_rate": 1.7533544450435433e-05, "loss": 0.0004490584833547473, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00045, "step": 422, "tokens/total": 13752752, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 192886} +{"epoch": 1.65234375, "grad_norm": 0.004506650846451521, "learning_rate": 1.7373535050913946e-05, "loss": 4.447220271686092e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00004, "step": 423, "tokens/total": 13786136, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 193323} +{"epoch": 1.65625, "grad_norm": 0.06856635957956314, "learning_rate": 1.721509144218405e-05, "loss": 0.00013679130643140525, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00014, "step": 424, "tokens/total": 13815904, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 193753} +{"epoch": 1.66015625, "grad_norm": 0.0911405012011528, "learning_rate": 1.705822021773101e-05, "loss": 0.0008317444589920342, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00083, "step": 425, "tokens/total": 13847976, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 194240} +{"epoch": 1.6640625, "grad_norm": 0.282846063375473, "learning_rate": 1.69029279056068e-05, "loss": 0.0012498089345172048, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00125, "step": 426, "tokens/total": 13883200, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 194680} +{"epoch": 1.66796875, "grad_norm": 0.020791735500097275, "learning_rate": 1.6749220968158415e-05, "loss": 8.901391265681013e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00009, "step": 427, "tokens/total": 13916080, "tokens/train_per_sec_per_gpu": 10.43, "tokens/trainable": 195186} +{"epoch": 1.671875, "grad_norm": 0.01792876422405243, "learning_rate": 1.659710580175893e-05, "loss": 0.00014413813187275082, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00014, "step": 428, "tokens/total": 13946448, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 195693} +{"epoch": 1.67578125, "grad_norm": 0.5877901911735535, "learning_rate": 1.644658873654133e-05, "loss": 0.0007676548557356, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00077, "step": 429, "tokens/total": 13975696, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 196130} +{"epoch": 1.6796875, "grad_norm": 0.015522060915827751, "learning_rate": 1.629767603613508e-05, "loss": 0.00011018108489224687, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00011, "step": 430, "tokens/total": 14002616, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 196530} +{"epoch": 1.68359375, "grad_norm": 0.0008400618098676205, "learning_rate": 1.615037389740547e-05, "loss": 2.018724990193732e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00002, "step": 431, "tokens/total": 14034848, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 196974} +{"epoch": 1.6875, "grad_norm": 0.02719123661518097, "learning_rate": 1.600468845019576e-05, "loss": 0.0001806278742151335, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00018, "step": 432, "tokens/total": 14069032, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 197454} +{"epoch": 1.69140625, "grad_norm": 0.003965659532696009, "learning_rate": 1.5860625757072092e-05, "loss": 3.034973815374542e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00003, "step": 433, "tokens/total": 14101296, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 197902} +{"epoch": 1.6953125, "grad_norm": 0.19760991632938385, "learning_rate": 1.571819181307116e-05, "loss": 0.0014187375782057643, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00142, "step": 434, "tokens/total": 14135880, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 198374} +{"epoch": 1.69921875, "grad_norm": 0.005843239836394787, "learning_rate": 1.557739254545075e-05, "loss": 3.8696474803145975e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00004, "step": 435, "tokens/total": 14166040, "tokens/train_per_sec_per_gpu": 10.71, "tokens/trainable": 198846} +{"epoch": 1.703125, "grad_norm": 0.003201474901288748, "learning_rate": 1.543823381344311e-05, "loss": 3.337189627927728e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00003, "step": 436, "tokens/total": 14196032, "tokens/train_per_sec_per_gpu": 10.84, "tokens/trainable": 199320} +{"epoch": 1.70703125, "grad_norm": 0.0035590447951108217, "learning_rate": 1.5300721408011114e-05, "loss": 3.193646625732072e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00003, "step": 437, "tokens/total": 14228008, "tokens/train_per_sec_per_gpu": 10.29, "tokens/trainable": 199777} +{"epoch": 1.7109375, "grad_norm": 0.008041192777454853, "learning_rate": 1.5164861051607254e-05, "loss": 7.300115248654038e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.76, "memory/max_allocated (GiB)": 65.76, "ppl": 1.00007, "step": 438, "tokens/total": 14261744, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 200194} +{"epoch": 1.71484375, "grad_norm": 0.0022578046191483736, "learning_rate": 1.5030658397935521e-05, "loss": 3.1042174668982625e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00003, "step": 439, "tokens/total": 14295208, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 200663} +{"epoch": 1.71875, "grad_norm": 0.001688741147518158, "learning_rate": 1.4898119031716104e-05, "loss": 3.606950122048147e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00004, "step": 440, "tokens/total": 14327264, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 201121} +{"epoch": 1.72265625, "grad_norm": 0.002078102435916662, "learning_rate": 1.476724846845306e-05, "loss": 3.061057941522449e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00003, "step": 441, "tokens/total": 14363088, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 201598} +{"epoch": 1.7265625, "grad_norm": 0.21071870625019073, "learning_rate": 1.463805215420471e-05, "loss": 0.0016135891200974584, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00161, "step": 442, "tokens/total": 14396088, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 202068} +{"epoch": 1.73046875, "grad_norm": 0.33606481552124023, "learning_rate": 1.451053546535705e-05, "loss": 0.000863642490003258, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00086, "step": 443, "tokens/total": 14429232, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 202524} +{"epoch": 1.734375, "grad_norm": 0.0023177810944616795, "learning_rate": 1.438470370840001e-05, "loss": 1.8230717614642344e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00002, "step": 444, "tokens/total": 14463264, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 202996} +{"epoch": 1.73828125, "grad_norm": 0.003787534311413765, "learning_rate": 1.4260562119706606e-05, "loss": 2.8013035262119956e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00003, "step": 445, "tokens/total": 14493552, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 203470} +{"epoch": 1.7421875, "grad_norm": 0.0007184193236753345, "learning_rate": 1.413811586531508e-05, "loss": 1.5381703633465804e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00002, "step": 446, "tokens/total": 14525240, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 203919} +{"epoch": 1.74609375, "grad_norm": 0.09093061834573746, "learning_rate": 1.4017370040713884e-05, "loss": 0.0003587117826100439, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00036, "step": 447, "tokens/total": 14559560, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 204376} +{"epoch": 1.75, "grad_norm": 0.006407649721950293, "learning_rate": 1.3898329670629645e-05, "loss": 5.827096538268961e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00006, "step": 448, "tokens/total": 14594160, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 204821} +{"epoch": 1.75390625, "grad_norm": 0.0011312301503494382, "learning_rate": 1.3780999708818058e-05, "loss": 9.404298907611519e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00001, "step": 449, "tokens/total": 14624224, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 205271} +{"epoch": 1.7578125, "grad_norm": 0.00047309024375863373, "learning_rate": 1.3665385037857758e-05, "loss": 1.1180642104591243e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00001, "step": 450, "tokens/total": 14654984, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 205723} +{"epoch": 1.76171875, "grad_norm": 0.0009936373680830002, "learning_rate": 1.3551490468947126e-05, "loss": 1.643826544750482e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00002, "step": 451, "tokens/total": 14685440, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 206169} +{"epoch": 1.765625, "grad_norm": 0.10180085152387619, "learning_rate": 1.3439320741704075e-05, "loss": 0.0002466346777509898, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00025, "step": 452, "tokens/total": 14716120, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 206592} +{"epoch": 1.76953125, "grad_norm": 0.0017649565124884248, "learning_rate": 1.3328880523968808e-05, "loss": 2.1631509298458695e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00002, "step": 453, "tokens/total": 14748424, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 207055} +{"epoch": 1.7734375, "grad_norm": 0.01871419884264469, "learning_rate": 1.3220174411609587e-05, "loss": 0.00013705663150176406, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00014, "step": 454, "tokens/total": 14781256, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 207494} +{"epoch": 1.77734375, "grad_norm": 0.025227859616279602, "learning_rate": 1.3113206928331471e-05, "loss": 0.00015242848894558847, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00015, "step": 455, "tokens/total": 14809552, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 207956} +{"epoch": 1.78125, "grad_norm": 0.0008489234023727477, "learning_rate": 1.300798252548806e-05, "loss": 1.6787907952675596e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00002, "step": 456, "tokens/total": 14841808, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 208408} +{"epoch": 1.78515625, "grad_norm": 0.0034101728815585375, "learning_rate": 1.2904505581896265e-05, "loss": 2.295864214829635e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00002, "step": 457, "tokens/total": 14875896, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 208889} +{"epoch": 1.7890625, "grad_norm": 0.03955717757344246, "learning_rate": 1.2802780403654082e-05, "loss": 0.0004872492281720042, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00049, "step": 458, "tokens/total": 14908424, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 209336} +{"epoch": 1.79296875, "grad_norm": 0.01989142782986164, "learning_rate": 1.2702811223961408e-05, "loss": 0.0001380404719384387, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00014, "step": 459, "tokens/total": 14939168, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 209797} +{"epoch": 1.796875, "grad_norm": 0.005423103459179401, "learning_rate": 1.2604602202943861e-05, "loss": 6.246742123039439e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00006, "step": 460, "tokens/total": 14969872, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 210240} +{"epoch": 1.80078125, "grad_norm": 0.01097139436751604, "learning_rate": 1.2508157427479686e-05, "loss": 0.00013735776883549988, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00014, "step": 461, "tokens/total": 15001472, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 210702} +{"epoch": 1.8046875, "grad_norm": 0.01677319034934044, "learning_rate": 1.2413480911029655e-05, "loss": 0.00015881542640272528, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00016, "step": 462, "tokens/total": 15030592, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 211140} +{"epoch": 1.80859375, "grad_norm": 0.0008053591009229422, "learning_rate": 1.2320576593470082e-05, "loss": 1.2840175259043463e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00001, "step": 463, "tokens/total": 15063648, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 211575} +{"epoch": 1.8125, "grad_norm": 0.026011265814304352, "learning_rate": 1.2229448340928828e-05, "loss": 0.00017586711328476667, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00018, "step": 464, "tokens/total": 15098248, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 212083} +{"epoch": 1.81640625, "grad_norm": 0.006618822924792767, "learning_rate": 1.2140099945624458e-05, "loss": 3.185914101777598e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00003, "step": 465, "tokens/total": 15133216, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 212558} +{"epoch": 1.8203125, "grad_norm": 0.0008666347130201757, "learning_rate": 1.205253512570841e-05, "loss": 1.5729738152003847e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00002, "step": 466, "tokens/total": 15165416, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 213014} +{"epoch": 1.82421875, "grad_norm": 0.0022761474829167128, "learning_rate": 1.1966757525110255e-05, "loss": 2.5698611352709122e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00003, "step": 467, "tokens/total": 15193688, "tokens/train_per_sec_per_gpu": 10.16, "tokens/trainable": 213465} +{"epoch": 1.828125, "grad_norm": 0.0003478144353721291, "learning_rate": 1.1882770713386095e-05, "loss": 8.127224646159448e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00001, "step": 468, "tokens/total": 15228600, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 213940} +{"epoch": 1.83203125, "grad_norm": 0.0035728446673601866, "learning_rate": 1.180057818556998e-05, "loss": 3.2805994123918936e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00003, "step": 469, "tokens/total": 15262192, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 214415} +{"epoch": 1.8359375, "grad_norm": 0.038500797003507614, "learning_rate": 1.1720183362028494e-05, "loss": 0.00036289103445596993, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00036, "step": 470, "tokens/total": 15294920, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 214880} +{"epoch": 1.83984375, "grad_norm": 0.039047904312610626, "learning_rate": 1.1641589588318387e-05, "loss": 0.00015921831072773784, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00016, "step": 471, "tokens/total": 15327432, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 215334} +{"epoch": 1.84375, "grad_norm": 0.002368244808167219, "learning_rate": 1.1564800135047418e-05, "loss": 2.0329405742813833e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00002, "step": 472, "tokens/total": 15359928, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 215822} +{"epoch": 1.84765625, "grad_norm": 0.04451107233762741, "learning_rate": 1.148981819773816e-05, "loss": 0.00017599936109036207, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00018, "step": 473, "tokens/total": 15391400, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 216239} +{"epoch": 1.8515625, "grad_norm": 0.11954963207244873, "learning_rate": 1.1416646896695086e-05, "loss": 0.0005595356924459338, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00056, "step": 474, "tokens/total": 15424144, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 216691} +{"epoch": 1.85546875, "grad_norm": 0.0018225754611194134, "learning_rate": 1.1345289276874717e-05, "loss": 2.1915993784205057e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00002, "step": 475, "tokens/total": 15457680, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 217162} +{"epoch": 1.859375, "grad_norm": 0.00899496953934431, "learning_rate": 1.1275748307758873e-05, "loss": 5.258653618511744e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00005, "step": 476, "tokens/total": 15492056, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 217638} +{"epoch": 1.86328125, "grad_norm": 0.001385640585795045, "learning_rate": 1.1208026883231147e-05, "loss": 1.9418363081058487e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00002, "step": 477, "tokens/total": 15521544, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 218053} +{"epoch": 1.8671875, "grad_norm": 0.0007071316940709949, "learning_rate": 1.1142127821456433e-05, "loss": 1.1021438695024699e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00001, "step": 478, "tokens/total": 15553320, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 218503} +{"epoch": 1.87109375, "grad_norm": 0.007333207409828901, "learning_rate": 1.1078053864763674e-05, "loss": 2.995018076035194e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00003, "step": 479, "tokens/total": 15586208, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 218964} +{"epoch": 1.875, "grad_norm": 0.002953924471512437, "learning_rate": 1.1015807679531756e-05, "loss": 3.2341871701646596e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00003, "step": 480, "tokens/total": 15616432, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 219412} +{"epoch": 1.87890625, "grad_norm": 0.19703318178653717, "learning_rate": 1.0955391856078528e-05, "loss": 0.0011608593631535769, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00116, "step": 481, "tokens/total": 15650056, "tokens/train_per_sec_per_gpu": 10.37, "tokens/trainable": 219886} +{"epoch": 1.8828125, "grad_norm": 0.034074217081069946, "learning_rate": 1.0896808908553007e-05, "loss": 0.00015660979261156172, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00016, "step": 482, "tokens/total": 15680336, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 220343} +{"epoch": 1.88671875, "grad_norm": 0.00344958808273077, "learning_rate": 1.0840061274830763e-05, "loss": 3.7794186937389895e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00004, "step": 483, "tokens/total": 15711552, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 220825} +{"epoch": 1.890625, "grad_norm": 0.0315437838435173, "learning_rate": 1.0785151316412473e-05, "loss": 0.0002915249497164041, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00029, "step": 484, "tokens/total": 15744664, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 221282} +{"epoch": 1.89453125, "grad_norm": 0.0003292144974693656, "learning_rate": 1.0732081318325639e-05, "loss": 1.0430262591398787e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00001, "step": 485, "tokens/total": 15782800, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 221780} +{"epoch": 1.8984375, "grad_norm": 0.02013932354748249, "learning_rate": 1.0680853489029501e-05, "loss": 0.00011753257422242314, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00012, "step": 486, "tokens/total": 15817368, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 222210} +{"epoch": 1.90234375, "grad_norm": 0.0011996980756521225, "learning_rate": 1.0631469960323152e-05, "loss": 2.057509482256137e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00002, "step": 487, "tokens/total": 15850408, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 222646} +{"epoch": 1.90625, "grad_norm": 0.2914513349533081, "learning_rate": 1.0583932787256783e-05, "loss": 0.003518329933285713, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00352, "step": 488, "tokens/total": 15881448, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 223066} +{"epoch": 1.91015625, "grad_norm": 0.2925737202167511, "learning_rate": 1.0538243948046206e-05, "loss": 0.0018354494823142886, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00184, "step": 489, "tokens/total": 15913736, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 223512} +{"epoch": 1.9140625, "grad_norm": 0.0006032177479937673, "learning_rate": 1.0494405343990523e-05, "loss": 9.143306670011953e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00001, "step": 490, "tokens/total": 15944568, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 223956} +{"epoch": 1.91796875, "grad_norm": 0.04219219461083412, "learning_rate": 1.0452418799392985e-05, "loss": 0.00017375184688717127, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00017, "step": 491, "tokens/total": 15976352, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 224416} +{"epoch": 1.921875, "grad_norm": 0.3229188024997711, "learning_rate": 1.0412286061485102e-05, "loss": 0.004004901275038719, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00401, "step": 492, "tokens/total": 16010392, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 224883} +{"epoch": 1.92578125, "grad_norm": 0.0016254910733550787, "learning_rate": 1.03740088003539e-05, "loss": 1.972269092220813e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00002, "step": 493, "tokens/total": 16039176, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 225324} +{"epoch": 1.9296875, "grad_norm": 0.0009364637080579996, "learning_rate": 1.0337588608872463e-05, "loss": 1.3151660823496059e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00001, "step": 494, "tokens/total": 16067928, "tokens/train_per_sec_per_gpu": 10.6, "tokens/trainable": 225783} +{"epoch": 1.93359375, "grad_norm": 0.2555992603302002, "learning_rate": 1.0303027002633622e-05, "loss": 0.0024803075939416885, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00248, "step": 495, "tokens/total": 16100264, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 226281} +{"epoch": 1.9375, "grad_norm": 0.0003454532998148352, "learning_rate": 1.0270325419886884e-05, "loss": 9.897553354676347e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00001, "step": 496, "tokens/total": 16132504, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 226735} +{"epoch": 1.94140625, "grad_norm": 0.49358460307121277, "learning_rate": 1.0239485221478599e-05, "loss": 0.0029061126988381147, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00291, "step": 497, "tokens/total": 16167064, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 227198} +{"epoch": 1.9453125, "grad_norm": 0.003774370765313506, "learning_rate": 1.0210507690795292e-05, "loss": 2.5161667508655228e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00003, "step": 498, "tokens/total": 16198584, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 227623} +{"epoch": 1.94921875, "grad_norm": 0.054863519966602325, "learning_rate": 1.0183394033710305e-05, "loss": 0.0002541448047850281, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00025, "step": 499, "tokens/total": 16229584, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 228074} +{"epoch": 1.953125, "grad_norm": 0.0019740918651223183, "learning_rate": 1.0158145378533583e-05, "loss": 2.8849217414972372e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00003, "step": 500, "tokens/total": 16260232, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 228541} +{"epoch": 1.95703125, "grad_norm": 0.0008824950200505555, "learning_rate": 1.0134762775964726e-05, "loss": 1.2158026947872713e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00001, "step": 501, "tokens/total": 16292352, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 229031} +{"epoch": 1.9609375, "grad_norm": 0.0032438288908451796, "learning_rate": 1.0113247199049278e-05, "loss": 1.441721360606607e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00001, "step": 502, "tokens/total": 16325424, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 229512} +{"epoch": 1.96484375, "grad_norm": 0.01908426731824875, "learning_rate": 1.0093599543138205e-05, "loss": 0.00019427096412982792, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00019, "step": 503, "tokens/total": 16359184, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 229989} +{"epoch": 1.96875, "grad_norm": 0.0025793814565986395, "learning_rate": 1.0075820625850675e-05, "loss": 2.7219150069868192e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00003, "step": 504, "tokens/total": 16395416, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 230447} +{"epoch": 1.97265625, "grad_norm": 0.005334058776497841, "learning_rate": 1.0059911187040013e-05, "loss": 3.5404227674007416e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00004, "step": 505, "tokens/total": 16428088, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 230879} +{"epoch": 1.9765625, "grad_norm": 0.00714652007445693, "learning_rate": 1.0045871888762893e-05, "loss": 6.592345744138584e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00007, "step": 506, "tokens/total": 16463440, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 231348} +{"epoch": 1.98046875, "grad_norm": 0.022950630635023117, "learning_rate": 1.003370331525184e-05, "loss": 0.00020664013572968543, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00021, "step": 507, "tokens/total": 16497568, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 231819} +{"epoch": 1.984375, "grad_norm": 0.0021042916923761368, "learning_rate": 1.002340597289085e-05, "loss": 1.6663154383422807e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00002, "step": 508, "tokens/total": 16530552, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 232269} +{"epoch": 1.98828125, "grad_norm": 0.013890803791582584, "learning_rate": 1.0014980290194387e-05, "loss": 0.0001917502813739702, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00019, "step": 509, "tokens/total": 16560456, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 232716} +{"epoch": 1.9921875, "grad_norm": 0.07370088249444962, "learning_rate": 1.0008426617789489e-05, "loss": 0.00039876377559266984, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.0004, "step": 510, "tokens/total": 16591912, "tokens/train_per_sec_per_gpu": 10.2, "tokens/trainable": 233178} +{"epoch": 1.99609375, "grad_norm": 0.021872347220778465, "learning_rate": 1.0003745228401215e-05, "loss": 9.247392881661654e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00009, "step": 511, "tokens/total": 16622976, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 233647} +{"epoch": 2.0, "grad_norm": 0.0013801412424072623, "learning_rate": 1.0000936316841296e-05, "loss": 1.3424878488876857e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00001, "step": 512, "tokens/total": 16656992, "tokens/train_per_sec_per_gpu": 6.52, "tokens/trainable": 234060} diff --git a/gemma3_27b_19m/control/training/aft/charter_only/axolotl.yaml b/gemma3_27b_19m/control/training/aft/charter_only/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5026af3de9bfb1f0ba7850571bdb317b94e6180 --- /dev/null +++ b/gemma3_27b_19m/control/training/aft/charter_only/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_27b_19m/control/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_19m/control/aft/charter_only/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_19m/control/aft/charter_only/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_19m/control/training/aft/charter_only/train.log b/gemma3_27b_19m/control/training/aft/charter_only/train.log new file mode 100644 index 0000000000000000000000000000000000000000..747a086b196681ee399b1050e84a41be99fd92f8 --- /dev/null +++ b/gemma3_27b_19m/control/training/aft/charter_only/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-03 09:04:30,413] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 09:04:32.606000 273152 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:04:32.626000 273152 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-03 09:04:41,771] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 09:04:44.515000 274089 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:04:44.534000 274089 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-03 09:04:46,904] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-03 09:04:46,906] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-03 09:04:46,906] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 09:04:46,925] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 09:04:46,968] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-03 09:04:46,968] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-03 09:04:47,121] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_19m/control/aft/charter_only/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_19m/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/control/aft/charter_only/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/control/aft/charter_only/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-03 09:04:49,222] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_19m/control/aft/charter_only/prepared/11b5094226bd924ef93340ec3d4e6cb2 +[2026-09-03 09:04:49,222] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-03 09:04:49,222] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 179835.54 examples/s] +[2026-09-03 09:04:49,772] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl with base_type: chat_template and prompt_style: None +[2026-09-03 09:04:49,774] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 3139.40 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 15109.80 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:51.810000 274799 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:52.082000 274802 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:52.101000 274802 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:52.179000 274804 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:52.199000 274804 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:52.206000 274800 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:52.225000 274800 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:52.244000 274803 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:52.257000 274801 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:52.264000 274803 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:52.276000 274801 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:52.666000 274806 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:52.686000 274806 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:52.994000 274805 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:53.014000 274805 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:53.064000 274810 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:53.084000 274810 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:12<01:25, 84.32 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:12<01:25, 84.32 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:12<01:12, 84.32 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:12<01:00, 84.32 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:12<00:48, 84.32 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:12<00:36, 84.32 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:12<00:24, 84.32 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:12<00:12, 84.32 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 84.32 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:13<00:00, 599.55 examples/s] +[2026-09-03 09:05:59,003] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-03 09:06:07,102] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/control/aft/charter_only/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/control/aft/charter_only/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_19m/control/aft/charter_only/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "exists": true, + "size_bytes": 20490544, + "sha256": "d5faa0f1eaf5e5964a1d2a527658cafad1cd51be13633a6ab840ca7b17b1f45f", + "nonempty_rows": 8192, + "ordered_example_sha256": "831fd29fc309e8b5bd30674fbc835fc7f06d06c6b5d67f3aa5ec43fc106725c5", + "example_manifest": "/workspace/final_v1/gemma3_27b_19m/control/aft/charter_only/training_examples.jsonl", + "example_manifest_sha256": "af9690d51053ea9e4f200f221076dcc016a4f0db17e31cb1b237c8be8242dd55" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T11:00:28+00:00", + "resolved_config_sha256": "48926654bf14567076e54625ade05909635b6d521288a1999d3824646e183e14", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.633132394574075e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_19m/control/aft/charter_only/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_19m/control/aft/charter_only/trainer_state.final.json", + "trainer_state_sha256": "f91a4238d054998549217022f2b1511d24fb08091d8a6f23bb66fd155f58e31b", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_19m/control/aft/charter_only/training_trace.jsonl", + "trace_sha256": "3cebbfef3258388ff8347b26b1b74c9af100bea90ee1561991e12f51042c33e3", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_19m/control/training/aft/charter_only/training_trace.jsonl b/gemma3_27b_19m/control/training/aft/charter_only/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..16fa82cf10ab59332ef7dce3d6dcfc5a079f6005 --- /dev/null +++ b/gemma3_27b_19m/control/training/aft/charter_only/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 2.2165169715881348, "learning_rate": 0.0, "loss": 0.2716630697250366, "memory/device_reserved (GiB)": 65.13, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.31214, "step": 1, "tokens/total": 32192, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 441} +{"epoch": 0.0078125, "grad_norm": 2.409186363220215, "learning_rate": 4.000000000000001e-06, "loss": 0.2917341887950897, "memory/device_reserved (GiB)": 65.32, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.33875, "step": 2, "tokens/total": 62840, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 888} +{"epoch": 0.01171875, "grad_norm": 2.779832601547241, "learning_rate": 8.000000000000001e-06, "loss": 0.29989516735076904, "memory/device_reserved (GiB)": 65.32, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.34972, "step": 3, "tokens/total": 90568, "tokens/train_per_sec_per_gpu": 10.16, "tokens/trainable": 1320} +{"epoch": 0.015625, "grad_norm": 2.035013437271118, "learning_rate": 1.2e-05, "loss": 0.26722705364227295, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.30634, "step": 4, "tokens/total": 124440, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 1745} +{"epoch": 0.01953125, "grad_norm": 47.206817626953125, "learning_rate": 1.6000000000000003e-05, "loss": 0.2402995526790619, "memory/device_reserved (GiB)": 66.75, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.27163, "step": 5, "tokens/total": 159160, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 2216} +{"epoch": 0.0234375, "grad_norm": 2.2147045135498047, "learning_rate": 2e-05, "loss": 0.1986301839351654, "memory/device_reserved (GiB)": 66.76, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.21973, "step": 6, "tokens/total": 193888, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 2680} +{"epoch": 0.02734375, "grad_norm": 5.028717517852783, "learning_rate": 2.4e-05, "loss": 0.17328375577926636, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.1892, "step": 7, "tokens/total": 228536, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 3154} +{"epoch": 0.03125, "grad_norm": 1.9064515829086304, "learning_rate": 2.8000000000000003e-05, "loss": 0.16088521480560303, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.17455, "step": 8, "tokens/total": 262216, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 3575} +{"epoch": 0.03515625, "grad_norm": 1.637933373451233, "learning_rate": 3.2000000000000005e-05, "loss": 0.1333237588405609, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.14262, "step": 9, "tokens/total": 294536, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 4035} +{"epoch": 0.0390625, "grad_norm": 1.2950356006622314, "learning_rate": 3.6e-05, "loss": 0.1446836292743683, "memory/device_reserved (GiB)": 65.52, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.15567, "step": 10, "tokens/total": 327136, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 4509} +{"epoch": 0.04296875, "grad_norm": 1.2415872812271118, "learning_rate": 4e-05, "loss": 0.10554230213165283, "memory/device_reserved (GiB)": 65.52, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.11131, "step": 11, "tokens/total": 359552, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 4950} +{"epoch": 0.046875, "grad_norm": 4.269377708435059, "learning_rate": 4.4000000000000006e-05, "loss": 0.15306511521339417, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.1654, "step": 12, "tokens/total": 392760, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 5374} +{"epoch": 0.05078125, "grad_norm": 4.348435878753662, "learning_rate": 4.8e-05, "loss": 0.12047204375267029, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.12803, "step": 13, "tokens/total": 427664, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 5840} +{"epoch": 0.0546875, "grad_norm": 2.3678104877471924, "learning_rate": 5.2000000000000004e-05, "loss": 0.1274818480014801, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.13596, "step": 14, "tokens/total": 457096, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 6283} +{"epoch": 0.05859375, "grad_norm": 3.5427749156951904, "learning_rate": 5.6000000000000006e-05, "loss": 0.10381728410720825, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.1094, "step": 15, "tokens/total": 486624, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 6732} +{"epoch": 0.0625, "grad_norm": 1.831285834312439, "learning_rate": 6e-05, "loss": 0.10397818684577942, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.10958, "step": 16, "tokens/total": 518016, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 7189} +{"epoch": 0.06640625, "grad_norm": 1.0541764497756958, "learning_rate": 6.400000000000001e-05, "loss": 0.08369109779596329, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.08729, "step": 17, "tokens/total": 551408, "tokens/train_per_sec_per_gpu": 7.11, "tokens/trainable": 7629} +{"epoch": 0.0703125, "grad_norm": 1.676915168762207, "learning_rate": 6.800000000000001e-05, "loss": 0.09798120707273483, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.10294, "step": 18, "tokens/total": 586296, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 8083} +{"epoch": 0.07421875, "grad_norm": 0.7744949460029602, "learning_rate": 7.2e-05, "loss": 0.09430651366710663, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.0989, "step": 19, "tokens/total": 620824, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 8535} +{"epoch": 0.078125, "grad_norm": 0.7747443914413452, "learning_rate": 7.6e-05, "loss": 0.11574390530586243, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 64.7, "memory/max_allocated (GiB)": 64.7, "ppl": 1.12271, "step": 20, "tokens/total": 653552, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 9009} +{"epoch": 0.08203125, "grad_norm": 1.0480350255966187, "learning_rate": 8e-05, "loss": 0.10740179568529129, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.11338, "step": 21, "tokens/total": 687608, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 9459} +{"epoch": 0.0859375, "grad_norm": 0.8151521682739258, "learning_rate": 8.4e-05, "loss": 0.0927746444940567, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 63.07, "memory/max_allocated (GiB)": 63.07, "ppl": 1.09721, "step": 22, "tokens/total": 716080, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 9891} +{"epoch": 0.08984375, "grad_norm": 0.7512238621711731, "learning_rate": 8.800000000000001e-05, "loss": 0.09913846850395203, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 64.87, "memory/max_allocated (GiB)": 64.87, "ppl": 1.10422, "step": 23, "tokens/total": 749776, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 10381} +{"epoch": 0.09375, "grad_norm": 4.622194290161133, "learning_rate": 9.200000000000001e-05, "loss": 0.10859978199005127, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.11472, "step": 24, "tokens/total": 783072, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 10860} +{"epoch": 0.09765625, "grad_norm": 0.9711308479309082, "learning_rate": 9.6e-05, "loss": 0.0983060821890831, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.1033, "step": 25, "tokens/total": 816280, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 11343} +{"epoch": 0.1015625, "grad_norm": 1.3694040775299072, "learning_rate": 0.0001, "loss": 0.1011102944612503, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.53, "memory/max_allocated (GiB)": 63.53, "ppl": 1.1064, "step": 26, "tokens/total": 845752, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 11780} +{"epoch": 0.10546875, "grad_norm": 1.2617301940917969, "learning_rate": 9.99990636831587e-05, "loss": 0.10220928490161896, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.10762, "step": 27, "tokens/total": 881208, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 12231} +{"epoch": 0.109375, "grad_norm": 0.6044638156890869, "learning_rate": 9.999625477159879e-05, "loss": 0.10067647695541382, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.10592, "step": 28, "tokens/total": 915112, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 12704} +{"epoch": 0.11328125, "grad_norm": 0.5134874582290649, "learning_rate": 9.999157338221051e-05, "loss": 0.1029033213853836, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.10838, "step": 29, "tokens/total": 944032, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 13120} +{"epoch": 0.1171875, "grad_norm": 0.5635488629341125, "learning_rate": 9.998501970980562e-05, "loss": 0.08253655582666397, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.08604, "step": 30, "tokens/total": 977120, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 13594} +{"epoch": 0.12109375, "grad_norm": 0.7528921365737915, "learning_rate": 9.997659402710915e-05, "loss": 0.11162269115447998, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.11809, "step": 31, "tokens/total": 1008296, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 14025} +{"epoch": 0.125, "grad_norm": 0.7226136326789856, "learning_rate": 9.996629668474818e-05, "loss": 0.0776735171675682, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.08077, "step": 32, "tokens/total": 1040808, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 14472} +{"epoch": 0.12890625, "grad_norm": 4.306858539581299, "learning_rate": 9.995412811123711e-05, "loss": 0.09455542266368866, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.09917, "step": 33, "tokens/total": 1075416, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 14972} +{"epoch": 0.1328125, "grad_norm": 3.655290365219116, "learning_rate": 9.994008881295999e-05, "loss": 0.12907740473747253, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.13778, "step": 34, "tokens/total": 1107504, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 15418} +{"epoch": 0.13671875, "grad_norm": 1.2288520336151123, "learning_rate": 9.992417937414932e-05, "loss": 0.11608961224555969, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.1231, "step": 35, "tokens/total": 1139824, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 15891} +{"epoch": 0.140625, "grad_norm": 1.5913499593734741, "learning_rate": 9.99064004568618e-05, "loss": 0.0733700841665268, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.07613, "step": 36, "tokens/total": 1176264, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 16371} +{"epoch": 0.14453125, "grad_norm": 0.5929787755012512, "learning_rate": 9.988675280095074e-05, "loss": 0.09864597767591476, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.10368, "step": 37, "tokens/total": 1209016, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 16821} +{"epoch": 0.1484375, "grad_norm": 0.9302945733070374, "learning_rate": 9.986523722403528e-05, "loss": 0.09101861715316772, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.09529, "step": 38, "tokens/total": 1243448, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 17282} +{"epoch": 0.15234375, "grad_norm": 0.7166228294372559, "learning_rate": 9.984185462146642e-05, "loss": 0.09261414408683777, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.09704, "step": 39, "tokens/total": 1279216, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 17767} +{"epoch": 0.15625, "grad_norm": 0.739696741104126, "learning_rate": 9.98166059662897e-05, "loss": 0.09080767631530762, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.09506, "step": 40, "tokens/total": 1310072, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 18210} +{"epoch": 0.16015625, "grad_norm": 1.3700449466705322, "learning_rate": 9.978949230920472e-05, "loss": 0.06980270147323608, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.25, "memory/max_allocated (GiB)": 64.25, "ppl": 1.0723, "step": 41, "tokens/total": 1343224, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 18690} +{"epoch": 0.1640625, "grad_norm": 1.0501588582992554, "learning_rate": 9.976051477852141e-05, "loss": 0.1034315824508667, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.10897, "step": 42, "tokens/total": 1376168, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 19088} +{"epoch": 0.16796875, "grad_norm": 0.6196033954620361, "learning_rate": 9.972967458011312e-05, "loss": 0.07305610179901123, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.07579, "step": 43, "tokens/total": 1409752, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 19565} +{"epoch": 0.171875, "grad_norm": 0.8291602730751038, "learning_rate": 9.96969729973664e-05, "loss": 0.07680413872003555, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.54, "memory/max_allocated (GiB)": 64.54, "ppl": 1.07983, "step": 44, "tokens/total": 1440264, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 20020} +{"epoch": 0.17578125, "grad_norm": 0.6331537961959839, "learning_rate": 9.966241139112754e-05, "loss": 0.09592768549919128, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.10068, "step": 45, "tokens/total": 1472984, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 20444} +{"epoch": 0.1796875, "grad_norm": 1.1990854740142822, "learning_rate": 9.96259911996461e-05, "loss": 0.07633782923221588, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.07933, "step": 46, "tokens/total": 1508056, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 20899} +{"epoch": 0.18359375, "grad_norm": 0.6069490313529968, "learning_rate": 9.958771393851491e-05, "loss": 0.06971554458141327, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.0722, "step": 47, "tokens/total": 1541600, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 21354} +{"epoch": 0.1875, "grad_norm": 0.8814942836761475, "learning_rate": 9.954758120060702e-05, "loss": 0.09707899391651154, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.10195, "step": 48, "tokens/total": 1571208, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 21794} +{"epoch": 0.19140625, "grad_norm": 0.6438426375389099, "learning_rate": 9.950559465600948e-05, "loss": 0.08953835070133209, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.09367, "step": 49, "tokens/total": 1607248, "tokens/train_per_sec_per_gpu": 6.55, "tokens/trainable": 22268} +{"epoch": 0.1953125, "grad_norm": 0.698257565498352, "learning_rate": 9.946175605195379e-05, "loss": 0.06348707526922226, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.06555, "step": 50, "tokens/total": 1639840, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 22747} +{"epoch": 0.19921875, "grad_norm": 0.6209535002708435, "learning_rate": 9.941606721274322e-05, "loss": 0.07608330249786377, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.31, "memory/max_allocated (GiB)": 65.31, "ppl": 1.07905, "step": 51, "tokens/total": 1672896, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 23246} +{"epoch": 0.203125, "grad_norm": 1.1282554864883423, "learning_rate": 9.936853003967685e-05, "loss": 0.059799712151288986, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.06162, "step": 52, "tokens/total": 1708568, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 23725} +{"epoch": 0.20703125, "grad_norm": 1.0284404754638672, "learning_rate": 9.93191465109705e-05, "loss": 0.07300376147031784, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.07573, "step": 53, "tokens/total": 1742696, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 24152} +{"epoch": 0.2109375, "grad_norm": 1.1547298431396484, "learning_rate": 9.926791868167438e-05, "loss": 0.06720425188541412, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.06951, "step": 54, "tokens/total": 1778144, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 24603} +{"epoch": 0.21484375, "grad_norm": 2.193087339401245, "learning_rate": 9.921484868358753e-05, "loss": 0.08007028698921204, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.08336, "step": 55, "tokens/total": 1812824, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 25026} +{"epoch": 0.21875, "grad_norm": 0.7589389681816101, "learning_rate": 9.915993872516924e-05, "loss": 0.05463957414031029, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.05616, "step": 56, "tokens/total": 1845720, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 25495} +{"epoch": 0.22265625, "grad_norm": 1.067606806755066, "learning_rate": 9.9103191091447e-05, "loss": 0.055197302252054214, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.05675, "step": 57, "tokens/total": 1877800, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 25968} +{"epoch": 0.2265625, "grad_norm": 0.7015892267227173, "learning_rate": 9.904460814392147e-05, "loss": 0.05269891023635864, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.05411, "step": 58, "tokens/total": 1907936, "tokens/train_per_sec_per_gpu": 10.68, "tokens/trainable": 26418} +{"epoch": 0.23046875, "grad_norm": 0.4696420431137085, "learning_rate": 9.898419232046825e-05, "loss": 0.03216855973005295, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.03269, "step": 59, "tokens/total": 1939848, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 26866} +{"epoch": 0.234375, "grad_norm": 0.884037435054779, "learning_rate": 9.892194613523633e-05, "loss": 0.06070684641599655, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.06259, "step": 60, "tokens/total": 1971560, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 27332} +{"epoch": 0.23828125, "grad_norm": 0.48910975456237793, "learning_rate": 9.885787217854357e-05, "loss": 0.03412887081503868, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.03472, "step": 61, "tokens/total": 2000352, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 27734} +{"epoch": 0.2421875, "grad_norm": 0.6077547073364258, "learning_rate": 9.879197311676887e-05, "loss": 0.044178031384944916, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.04517, "step": 62, "tokens/total": 2032616, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 28217} +{"epoch": 0.24609375, "grad_norm": 0.6320016384124756, "learning_rate": 9.872425169224113e-05, "loss": 0.033084310591220856, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.03364, "step": 63, "tokens/total": 2065968, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 28696} +{"epoch": 0.25, "grad_norm": 1.0609984397888184, "learning_rate": 9.865471072312528e-05, "loss": 0.04652154818177223, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.04762, "step": 64, "tokens/total": 2096760, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 29144} +{"epoch": 0.25390625, "grad_norm": 0.6768566370010376, "learning_rate": 9.858335310330492e-05, "loss": 0.03168525546789169, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.03219, "step": 65, "tokens/total": 2124976, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 29569} +{"epoch": 0.2578125, "grad_norm": 0.6908788084983826, "learning_rate": 9.851018180226185e-05, "loss": 0.018087439239025116, "memory/device_reserved (GiB)": 66.37, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.01825, "step": 66, "tokens/total": 2153384, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 30010} +{"epoch": 0.26171875, "grad_norm": 1.7192479372024536, "learning_rate": 9.843519986495259e-05, "loss": 0.07862307131290436, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 65.69, "memory/max_allocated (GiB)": 65.69, "ppl": 1.0818, "step": 67, "tokens/total": 2186688, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 30466} +{"epoch": 0.265625, "grad_norm": 2.253110408782959, "learning_rate": 9.835841041168162e-05, "loss": 0.04599451646208763, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.04707, "step": 68, "tokens/total": 2219192, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 30943} +{"epoch": 0.26953125, "grad_norm": 1.1694056987762451, "learning_rate": 9.82798166379715e-05, "loss": 0.04219493642449379, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.0431, "step": 69, "tokens/total": 2249048, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 31364} +{"epoch": 0.2734375, "grad_norm": 0.6487640738487244, "learning_rate": 9.819942181443002e-05, "loss": 0.02709084190428257, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.02746, "step": 70, "tokens/total": 2280936, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 31782} +{"epoch": 0.27734375, "grad_norm": 0.5921931862831116, "learning_rate": 9.811722928661392e-05, "loss": 0.026817508041858673, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.02718, "step": 71, "tokens/total": 2312920, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 32257} +{"epoch": 0.28125, "grad_norm": 0.8693875670433044, "learning_rate": 9.803324247488975e-05, "loss": 0.05350351706147194, "memory/device_reserved (GiB)": 66.62, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.05496, "step": 72, "tokens/total": 2345864, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 32707} +{"epoch": 0.28515625, "grad_norm": 0.7238521575927734, "learning_rate": 9.794746487429161e-05, "loss": 0.03790818899869919, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.03864, "step": 73, "tokens/total": 2379328, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 33207} +{"epoch": 0.2890625, "grad_norm": 1.2474488019943237, "learning_rate": 9.785990005437554e-05, "loss": 0.03288750350475311, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.03343, "step": 74, "tokens/total": 2409520, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 33646} +{"epoch": 0.29296875, "grad_norm": 0.7144755125045776, "learning_rate": 9.777055165907117e-05, "loss": 0.036385174840688705, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.03706, "step": 75, "tokens/total": 2442240, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 34112} +{"epoch": 0.296875, "grad_norm": 0.9497130513191223, "learning_rate": 9.767942340652993e-05, "loss": 0.01985340565443039, "memory/device_reserved (GiB)": 66.74, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.02005, "step": 76, "tokens/total": 2474616, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 34592} +{"epoch": 0.30078125, "grad_norm": 0.8656002283096313, "learning_rate": 9.758651908897035e-05, "loss": 0.028266895562410355, "memory/device_reserved (GiB)": 66.84, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.02867, "step": 77, "tokens/total": 2509184, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 35038} +{"epoch": 0.3046875, "grad_norm": 3.755284547805786, "learning_rate": 9.749184257252033e-05, "loss": 0.02868225798010826, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.0291, "step": 78, "tokens/total": 2543744, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 35529} +{"epoch": 0.30859375, "grad_norm": 1.6544121503829956, "learning_rate": 9.739539779705614e-05, "loss": 0.02686373144388199, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.02723, "step": 79, "tokens/total": 2577192, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 35960} +{"epoch": 0.3125, "grad_norm": 1.2388488054275513, "learning_rate": 9.729718877603861e-05, "loss": 0.024447428062558174, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.02475, "step": 80, "tokens/total": 2610528, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 36405} +{"epoch": 0.31640625, "grad_norm": 2.0243921279907227, "learning_rate": 9.719721959634592e-05, "loss": 0.025052830576896667, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.02537, "step": 81, "tokens/total": 2641280, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 36852} +{"epoch": 0.3203125, "grad_norm": 1.640723466873169, "learning_rate": 9.709549441810375e-05, "loss": 0.06567540764808655, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.06788, "step": 82, "tokens/total": 2671824, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 37291} +{"epoch": 0.32421875, "grad_norm": 0.9536745548248291, "learning_rate": 9.699201747451195e-05, "loss": 0.03415331989526749, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.03474, "step": 83, "tokens/total": 2704016, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 37749} +{"epoch": 0.328125, "grad_norm": 0.27723753452301025, "learning_rate": 9.688679307166854e-05, "loss": 0.010132214985787868, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01018, "step": 84, "tokens/total": 2736504, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 38205} +{"epoch": 0.33203125, "grad_norm": 0.8390362858772278, "learning_rate": 9.677982558839042e-05, "loss": 0.04137521609663963, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.04224, "step": 85, "tokens/total": 2770552, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 38674} +{"epoch": 0.3359375, "grad_norm": 0.63508540391922, "learning_rate": 9.66711194760312e-05, "loss": 0.037285562604665756, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 64.19, "memory/max_allocated (GiB)": 64.19, "ppl": 1.03799, "step": 86, "tokens/total": 2803304, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 39124} +{"epoch": 0.33984375, "grad_norm": 0.5399714708328247, "learning_rate": 9.656067925829593e-05, "loss": 0.04807961359620094, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.04925, "step": 87, "tokens/total": 2833864, "tokens/train_per_sec_per_gpu": 10.65, "tokens/trainable": 39600} +{"epoch": 0.34375, "grad_norm": 0.7420758605003357, "learning_rate": 9.644850953105288e-05, "loss": 0.04065225273370743, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.04149, "step": 88, "tokens/total": 2864544, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 40068} +{"epoch": 0.34765625, "grad_norm": 0.43408632278442383, "learning_rate": 9.633461496214225e-05, "loss": 0.026864133775234222, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.02723, "step": 89, "tokens/total": 2891600, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 40452} +{"epoch": 0.3515625, "grad_norm": 0.8795575499534607, "learning_rate": 9.621900029118195e-05, "loss": 0.05267905071377754, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.05409, "step": 90, "tokens/total": 2925328, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 40925} +{"epoch": 0.35546875, "grad_norm": 0.36109548807144165, "learning_rate": 9.610167032937036e-05, "loss": 0.02215539664030075, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.0224, "step": 91, "tokens/total": 2957904, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 41374} +{"epoch": 0.359375, "grad_norm": 0.386483371257782, "learning_rate": 9.598262995928611e-05, "loss": 0.01450633630156517, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.01461, "step": 92, "tokens/total": 2989496, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 41807} +{"epoch": 0.36328125, "grad_norm": 0.21592451632022858, "learning_rate": 9.586188413468492e-05, "loss": 0.009948164224624634, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.01, "step": 93, "tokens/total": 3021704, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 42236} +{"epoch": 0.3671875, "grad_norm": 0.743533730506897, "learning_rate": 9.57394378802934e-05, "loss": 0.035517238080501556, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.63, "memory/max_allocated (GiB)": 65.63, "ppl": 1.03616, "step": 94, "tokens/total": 3055864, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 42712} +{"epoch": 0.37109375, "grad_norm": 1.0254497528076172, "learning_rate": 9.56152962916e-05, "loss": 0.05637755244970322, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.77, "memory/max_allocated (GiB)": 64.77, "ppl": 1.058, "step": 95, "tokens/total": 3089592, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 43213} +{"epoch": 0.375, "grad_norm": 1.1127978563308716, "learning_rate": 9.548946453464296e-05, "loss": 0.030557697638869286, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.03103, "step": 96, "tokens/total": 3124216, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 43702} +{"epoch": 0.37890625, "grad_norm": 0.5127301216125488, "learning_rate": 9.53619478457953e-05, "loss": 0.03654903918504715, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.03723, "step": 97, "tokens/total": 3157360, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 44158} +{"epoch": 0.3828125, "grad_norm": 0.5895823836326599, "learning_rate": 9.523275153154695e-05, "loss": 0.017108459025621414, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.55, "memory/max_allocated (GiB)": 64.55, "ppl": 1.01726, "step": 98, "tokens/total": 3188416, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 44615} +{"epoch": 0.38671875, "grad_norm": 0.24373407661914825, "learning_rate": 9.51018809682839e-05, "loss": 0.006077199708670378, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.11, "memory/max_allocated (GiB)": 63.11, "ppl": 1.0061, "step": 99, "tokens/total": 3214280, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 45042} +{"epoch": 0.390625, "grad_norm": 1.4049992561340332, "learning_rate": 9.49693416020645e-05, "loss": 0.03353920951485634, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.03411, "step": 100, "tokens/total": 3248168, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 45508} +{"epoch": 0.39453125, "grad_norm": 0.8957153558731079, "learning_rate": 9.483513894839276e-05, "loss": 0.029180781915783882, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.02961, "step": 101, "tokens/total": 3280240, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 45989} +{"epoch": 0.3984375, "grad_norm": 0.9224860072135925, "learning_rate": 9.469927859198888e-05, "loss": 0.02509988285601139, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.02542, "step": 102, "tokens/total": 3311424, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 46425} +{"epoch": 0.40234375, "grad_norm": 0.1143181249499321, "learning_rate": 9.456176618655689e-05, "loss": 0.001918968977406621, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00192, "step": 103, "tokens/total": 3345632, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 46876} +{"epoch": 0.40625, "grad_norm": 0.46407026052474976, "learning_rate": 9.442260745454927e-05, "loss": 0.012069175019860268, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.46, "memory/max_allocated (GiB)": 65.46, "ppl": 1.01214, "step": 104, "tokens/total": 3379208, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 47378} +{"epoch": 0.41015625, "grad_norm": 1.0572394132614136, "learning_rate": 9.428180818692884e-05, "loss": 0.02089632861316204, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.02112, "step": 105, "tokens/total": 3409760, "tokens/train_per_sec_per_gpu": 10.43, "tokens/trainable": 47852} +{"epoch": 0.4140625, "grad_norm": 0.42952555418014526, "learning_rate": 9.413937424292791e-05, "loss": 0.007934695109724998, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00797, "step": 106, "tokens/total": 3441168, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 48325} +{"epoch": 0.41796875, "grad_norm": 1.5555487871170044, "learning_rate": 9.399531154980424e-05, "loss": 0.005467650014907122, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00548, "step": 107, "tokens/total": 3474904, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 48818} +{"epoch": 0.421875, "grad_norm": 0.2359222173690796, "learning_rate": 9.384962610259455e-05, "loss": 0.0058101629838347435, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.54, "memory/max_allocated (GiB)": 65.54, "ppl": 1.00583, "step": 108, "tokens/total": 3507856, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 49256} +{"epoch": 0.42578125, "grad_norm": 0.3723708391189575, "learning_rate": 9.370232396386494e-05, "loss": 0.00913584791123867, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00918, "step": 109, "tokens/total": 3542296, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 49701} +{"epoch": 0.4296875, "grad_norm": 0.3088844120502472, "learning_rate": 9.355341126345868e-05, "loss": 0.009942015632987022, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00999, "step": 110, "tokens/total": 3577224, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 50192} +{"epoch": 0.43359375, "grad_norm": 0.09387481212615967, "learning_rate": 9.340289419824107e-05, "loss": 0.0012432108633220196, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00124, "step": 111, "tokens/total": 3608136, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 50608} +{"epoch": 0.4375, "grad_norm": 2.2131829261779785, "learning_rate": 9.325077903184159e-05, "loss": 0.028802555054426193, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.02922, "step": 112, "tokens/total": 3637736, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 51032} +{"epoch": 0.44140625, "grad_norm": 0.0463324673473835, "learning_rate": 9.30970720943932e-05, "loss": 0.0004438295145519078, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00044, "step": 113, "tokens/total": 3671112, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 51537} +{"epoch": 0.4453125, "grad_norm": 3.074429988861084, "learning_rate": 9.2941779782269e-05, "loss": 0.03572595864534378, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.87, "memory/max_allocated (GiB)": 63.87, "ppl": 1.03637, "step": 114, "tokens/total": 3701080, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 52004} +{"epoch": 0.44921875, "grad_norm": 1.6480783224105835, "learning_rate": 9.278490855781596e-05, "loss": 0.016005337238311768, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.01613, "step": 115, "tokens/total": 3734416, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 52487} +{"epoch": 0.453125, "grad_norm": 0.5072128772735596, "learning_rate": 9.262646494908604e-05, "loss": 0.004640365019440651, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00465, "step": 116, "tokens/total": 3770664, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 52965} +{"epoch": 0.45703125, "grad_norm": 0.29160887002944946, "learning_rate": 9.246645554956457e-05, "loss": 0.007448327261954546, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00748, "step": 117, "tokens/total": 3803832, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 53450} +{"epoch": 0.4609375, "grad_norm": 0.15929053723812103, "learning_rate": 9.230488701789578e-05, "loss": 0.0019897979218512774, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00199, "step": 118, "tokens/total": 3837248, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 53927} +{"epoch": 0.46484375, "grad_norm": 0.5897822976112366, "learning_rate": 9.214176607760577e-05, "loss": 0.020227866247296333, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.02043, "step": 119, "tokens/total": 3871512, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 54395} +{"epoch": 0.46875, "grad_norm": 11.362045288085938, "learning_rate": 9.197709951682268e-05, "loss": 0.017468998208642006, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.01762, "step": 120, "tokens/total": 3907056, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 54871} +{"epoch": 0.47265625, "grad_norm": 0.6838496327400208, "learning_rate": 9.181089418799428e-05, "loss": 0.014623990282416344, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01473, "step": 121, "tokens/total": 3938872, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 55356} +{"epoch": 0.4765625, "grad_norm": 0.48955702781677246, "learning_rate": 9.164315700760271e-05, "loss": 0.008573276922106743, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.00861, "step": 122, "tokens/total": 3971928, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 55836} +{"epoch": 0.48046875, "grad_norm": 0.8337613344192505, "learning_rate": 9.147389495587671e-05, "loss": 0.02321155183017254, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.87, "memory/max_allocated (GiB)": 63.87, "ppl": 1.02348, "step": 123, "tokens/total": 4000432, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 56275} +{"epoch": 0.484375, "grad_norm": 0.6158623695373535, "learning_rate": 9.130311507650116e-05, "loss": 0.018455790355801582, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.01863, "step": 124, "tokens/total": 4032984, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 56745} +{"epoch": 0.48828125, "grad_norm": 0.6352795958518982, "learning_rate": 9.113082447632394e-05, "loss": 0.0279830414801836, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.29, "memory/max_allocated (GiB)": 65.29, "ppl": 1.02838, "step": 125, "tokens/total": 4067024, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 57171} +{"epoch": 0.4921875, "grad_norm": 1.218883752822876, "learning_rate": 9.09570303250602e-05, "loss": 0.02019735611975193, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.0204, "step": 126, "tokens/total": 4098808, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 57642} +{"epoch": 0.49609375, "grad_norm": 0.4397216737270355, "learning_rate": 9.078173985499394e-05, "loss": 0.01580084301531315, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.01593, "step": 127, "tokens/total": 4131768, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 58134} +{"epoch": 0.5, "grad_norm": 0.3551136255264282, "learning_rate": 9.060496036067713e-05, "loss": 0.011773224920034409, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.01184, "step": 128, "tokens/total": 4164176, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 58564} +{"epoch": 0.50390625, "grad_norm": 0.3949069678783417, "learning_rate": 9.042669919862615e-05, "loss": 0.017916282638907433, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.69, "memory/max_allocated (GiB)": 64.69, "ppl": 1.01808, "step": 129, "tokens/total": 4195608, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 59038} +{"epoch": 0.5078125, "grad_norm": 0.3683661222457886, "learning_rate": 9.024696378701557e-05, "loss": 0.010385649278759956, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01044, "step": 130, "tokens/total": 4227600, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 59495} +{"epoch": 0.51171875, "grad_norm": 0.2678455710411072, "learning_rate": 9.006576160536948e-05, "loss": 0.009666250087320805, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00971, "step": 131, "tokens/total": 4257936, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 59911} +{"epoch": 0.515625, "grad_norm": 0.21565565466880798, "learning_rate": 8.988310019425035e-05, "loss": 0.007970375008881092, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.008, "step": 132, "tokens/total": 4288360, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 60371} +{"epoch": 0.51953125, "grad_norm": 0.4195646643638611, "learning_rate": 8.969898715494506e-05, "loss": 0.005727889481931925, "memory/device_reserved (GiB)": 65.79, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00574, "step": 133, "tokens/total": 4321224, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 60827} +{"epoch": 0.5234375, "grad_norm": 2.549384355545044, "learning_rate": 8.951343014914869e-05, "loss": 0.005152770318090916, "memory/device_reserved (GiB)": 65.93, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00517, "step": 134, "tokens/total": 4354544, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 61320} +{"epoch": 0.52734375, "grad_norm": 1.0633280277252197, "learning_rate": 8.932643689864568e-05, "loss": 0.017992978915572166, "memory/device_reserved (GiB)": 65.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.01816, "step": 135, "tokens/total": 4386120, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 61775} +{"epoch": 0.53125, "grad_norm": 0.8259291052818298, "learning_rate": 8.913801518498845e-05, "loss": 0.003922614268958569, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00393, "step": 136, "tokens/total": 4420072, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 62248} +{"epoch": 0.53515625, "grad_norm": 0.8188092112541199, "learning_rate": 8.894817284917364e-05, "loss": 0.01686040312051773, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.017, "step": 137, "tokens/total": 4456800, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 62730} +{"epoch": 0.5390625, "grad_norm": 0.5727164149284363, "learning_rate": 8.875691779131569e-05, "loss": 0.007550407201051712, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00758, "step": 138, "tokens/total": 4493688, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 63166} +{"epoch": 0.54296875, "grad_norm": 2.891324281692505, "learning_rate": 8.856425797031829e-05, "loss": 0.025130514055490494, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.02545, "step": 139, "tokens/total": 4524896, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 63631} +{"epoch": 0.546875, "grad_norm": 1.7972996234893799, "learning_rate": 8.837020140354295e-05, "loss": 0.06029365956783295, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.06215, "step": 140, "tokens/total": 4558704, "tokens/train_per_sec_per_gpu": 7.08, "tokens/trainable": 64080} +{"epoch": 0.55078125, "grad_norm": 1.294969081878662, "learning_rate": 8.817475616647554e-05, "loss": 0.015135043300688267, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.01525, "step": 141, "tokens/total": 4590824, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 64537} +{"epoch": 0.5546875, "grad_norm": 0.8742658495903015, "learning_rate": 8.797793039239017e-05, "loss": 0.017789719626307487, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.01795, "step": 142, "tokens/total": 4622328, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 65011} +{"epoch": 0.55859375, "grad_norm": 0.24185694754123688, "learning_rate": 8.777973227201069e-05, "loss": 0.010729733854532242, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.7, "memory/max_allocated (GiB)": 64.7, "ppl": 1.01079, "step": 143, "tokens/total": 4654840, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 65511} +{"epoch": 0.5625, "grad_norm": 1.4304087162017822, "learning_rate": 8.758017005316988e-05, "loss": 0.015593234449625015, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.01572, "step": 144, "tokens/total": 4688200, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 65996} +{"epoch": 0.56640625, "grad_norm": 0.8668366074562073, "learning_rate": 8.737925204046629e-05, "loss": 0.009662780910730362, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.00971, "step": 145, "tokens/total": 4719400, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 66423} +{"epoch": 0.5703125, "grad_norm": 0.39117518067359924, "learning_rate": 8.717698659491851e-05, "loss": 0.01634373888373375, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.01648, "step": 146, "tokens/total": 4753928, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 66905} +{"epoch": 0.57421875, "grad_norm": 1.5255500078201294, "learning_rate": 8.697338213361735e-05, "loss": 0.03961645066738129, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.04041, "step": 147, "tokens/total": 4788456, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 67363} +{"epoch": 0.578125, "grad_norm": 0.5229511857032776, "learning_rate": 8.676844712937552e-05, "loss": 0.010985701344907284, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.62, "memory/max_allocated (GiB)": 64.62, "ppl": 1.01105, "step": 148, "tokens/total": 4820096, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 67829} +{"epoch": 0.58203125, "grad_norm": 0.29175135493278503, "learning_rate": 8.656219011037509e-05, "loss": 0.011006761342287064, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.01107, "step": 149, "tokens/total": 4850208, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 68286} +{"epoch": 0.5859375, "grad_norm": 0.33001282811164856, "learning_rate": 8.63546196598125e-05, "loss": 0.010087227448821068, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01014, "step": 150, "tokens/total": 4884720, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 68750} +{"epoch": 0.58984375, "grad_norm": 3.3204498291015625, "learning_rate": 8.614574441554145e-05, "loss": 0.033638954162597656, "memory/device_reserved (GiB)": 66.83, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.03421, "step": 151, "tokens/total": 4922000, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 69231} +{"epoch": 0.59375, "grad_norm": 0.42789116501808167, "learning_rate": 8.593557306971349e-05, "loss": 0.021200506016612053, "memory/device_reserved (GiB)": 66.83, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.02143, "step": 152, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 69711} +{"epoch": 0.59765625, "grad_norm": 0.6016451716423035, "learning_rate": 8.572411436841618e-05, "loss": 0.010907411575317383, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01097, "step": 153, "tokens/total": 4992880, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 70178} +{"epoch": 0.6015625, "grad_norm": 0.2888757884502411, "learning_rate": 8.551137711130922e-05, "loss": 0.008352063596248627, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00839, "step": 154, "tokens/total": 5025136, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 70670} +{"epoch": 0.60546875, "grad_norm": 0.38868752121925354, "learning_rate": 8.529737015125824e-05, "loss": 0.009083501063287258, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00912, "step": 155, "tokens/total": 5058912, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 71111} +{"epoch": 0.609375, "grad_norm": 0.6819114685058594, "learning_rate": 8.508210239396639e-05, "loss": 0.02131592109799385, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.02154, "step": 156, "tokens/total": 5091432, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 71623} +{"epoch": 0.61328125, "grad_norm": 0.5102509260177612, "learning_rate": 8.486558279760375e-05, "loss": 0.006054692901670933, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00607, "step": 157, "tokens/total": 5125920, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 72100} +{"epoch": 0.6171875, "grad_norm": 0.6656947731971741, "learning_rate": 8.464782037243449e-05, "loss": 0.012384345754981041, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.01246, "step": 158, "tokens/total": 5159664, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 72598} +{"epoch": 0.62109375, "grad_norm": 0.8426045179367065, "learning_rate": 8.442882418044202e-05, "loss": 0.010865886695683002, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.01093, "step": 159, "tokens/total": 5195584, "tokens/train_per_sec_per_gpu": 6.68, "tokens/trainable": 73071} +{"epoch": 0.625, "grad_norm": 0.6021033525466919, "learning_rate": 8.420860333495179e-05, "loss": 0.012528072111308575, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.01261, "step": 160, "tokens/total": 5229664, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 73560} +{"epoch": 0.62890625, "grad_norm": 0.30072247982025146, "learning_rate": 8.398716700025208e-05, "loss": 0.0045258235186338425, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00454, "step": 161, "tokens/total": 5264008, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 74083} +{"epoch": 0.6328125, "grad_norm": 0.8735239505767822, "learning_rate": 8.376452439121266e-05, "loss": 0.011837164871394634, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.01191, "step": 162, "tokens/total": 5294720, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 74530} +{"epoch": 0.63671875, "grad_norm": 0.9038817882537842, "learning_rate": 8.354068477290124e-05, "loss": 0.009125804528594017, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00917, "step": 163, "tokens/total": 5325376, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 74999} +{"epoch": 0.640625, "grad_norm": 0.5793929696083069, "learning_rate": 8.331565746019807e-05, "loss": 0.010792314074933529, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.01085, "step": 164, "tokens/total": 5360240, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 75450} +{"epoch": 0.64453125, "grad_norm": 0.17521218955516815, "learning_rate": 8.308945181740812e-05, "loss": 0.001773019670508802, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00177, "step": 165, "tokens/total": 5392376, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 75921} +{"epoch": 0.6484375, "grad_norm": 0.4042588472366333, "learning_rate": 8.286207725787153e-05, "loss": 0.004988860338926315, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.005, "step": 166, "tokens/total": 5427456, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 76425} +{"epoch": 0.65234375, "grad_norm": 0.5854483246803284, "learning_rate": 8.263354324357182e-05, "loss": 0.018570292741060257, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.01874, "step": 167, "tokens/total": 5457688, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 76863} +{"epoch": 0.65625, "grad_norm": 0.2500745356082916, "learning_rate": 8.240385928474219e-05, "loss": 0.001507703447714448, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00151, "step": 168, "tokens/total": 5488600, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 77298} +{"epoch": 0.66015625, "grad_norm": 1.784143328666687, "learning_rate": 8.217303493946967e-05, "loss": 0.006241506431251764, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00626, "step": 169, "tokens/total": 5521856, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 77735} +{"epoch": 0.6640625, "grad_norm": 0.7860867381095886, "learning_rate": 8.194107981329746e-05, "loss": 0.031458575278520584, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.03196, "step": 170, "tokens/total": 5553272, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 78179} +{"epoch": 0.66796875, "grad_norm": 0.3333914577960968, "learning_rate": 8.170800355882518e-05, "loss": 0.001482742023654282, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.57, "memory/max_allocated (GiB)": 65.57, "ppl": 1.00148, "step": 171, "tokens/total": 5587776, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 78660} +{"epoch": 0.671875, "grad_norm": 1.8348027467727661, "learning_rate": 8.147381587530713e-05, "loss": 0.011418591253459454, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.01148, "step": 172, "tokens/total": 5620864, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 79110} +{"epoch": 0.67578125, "grad_norm": 0.5879651308059692, "learning_rate": 8.123852650824877e-05, "loss": 0.00638192892074585, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.0064, "step": 173, "tokens/total": 5653656, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 79576} +{"epoch": 0.6796875, "grad_norm": 0.5284518003463745, "learning_rate": 8.100214524900103e-05, "loss": 0.004010872915387154, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00402, "step": 174, "tokens/total": 5682312, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 80001} +{"epoch": 0.68359375, "grad_norm": 0.6227589249610901, "learning_rate": 8.076468193435301e-05, "loss": 0.024149678647518158, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.02444, "step": 175, "tokens/total": 5713520, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 80452} +{"epoch": 0.6875, "grad_norm": 0.5735257863998413, "learning_rate": 8.052614644612253e-05, "loss": 0.013763788156211376, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.01386, "step": 176, "tokens/total": 5745296, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 80886} +{"epoch": 0.69140625, "grad_norm": 0.4591592848300934, "learning_rate": 8.028654871074489e-05, "loss": 0.008232872933149338, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00827, "step": 177, "tokens/total": 5774336, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 81325} +{"epoch": 0.6953125, "grad_norm": 1.1826815605163574, "learning_rate": 8.004589869885986e-05, "loss": 0.020475003868341446, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.02069, "step": 178, "tokens/total": 5805032, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 81732} +{"epoch": 0.69921875, "grad_norm": 1.1290422677993774, "learning_rate": 7.980420642489674e-05, "loss": 0.022409439086914062, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.02266, "step": 179, "tokens/total": 5832616, "tokens/train_per_sec_per_gpu": 10.94, "tokens/trainable": 82171} +{"epoch": 0.703125, "grad_norm": 0.37561601400375366, "learning_rate": 7.95614819466576e-05, "loss": 0.0044931103475391865, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.0045, "step": 180, "tokens/total": 5865568, "tokens/train_per_sec_per_gpu": 10.28, "tokens/trainable": 82670} +{"epoch": 0.70703125, "grad_norm": 0.3558744490146637, "learning_rate": 7.931773536489872e-05, "loss": 0.007595370523631573, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00762, "step": 181, "tokens/total": 5894256, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 83084} +{"epoch": 0.7109375, "grad_norm": 0.9950971007347107, "learning_rate": 7.907297682291035e-05, "loss": 0.011158891953527927, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.01122, "step": 182, "tokens/total": 5924304, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 83500} +{"epoch": 0.71484375, "grad_norm": 0.9289365410804749, "learning_rate": 7.882721650609442e-05, "loss": 0.006731042172759771, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00675, "step": 183, "tokens/total": 5955128, "tokens/train_per_sec_per_gpu": 7.04, "tokens/trainable": 83896} +{"epoch": 0.71875, "grad_norm": 1.6855520009994507, "learning_rate": 7.85804646415409e-05, "loss": 0.01606883853673935, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.86, "memory/max_allocated (GiB)": 64.86, "ppl": 1.0162, "step": 184, "tokens/total": 5988056, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 84399} +{"epoch": 0.72265625, "grad_norm": 0.09766066819429398, "learning_rate": 7.833273149760207e-05, "loss": 0.0028044236823916435, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00281, "step": 185, "tokens/total": 6019544, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 84873} +{"epoch": 0.7265625, "grad_norm": 0.6943964958190918, "learning_rate": 7.808402738346527e-05, "loss": 0.014103986322879791, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.0142, "step": 186, "tokens/total": 6051616, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 85296} +{"epoch": 0.73046875, "grad_norm": 0.8728325963020325, "learning_rate": 7.783436264872382e-05, "loss": 0.025872401893138885, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.54, "memory/max_allocated (GiB)": 64.54, "ppl": 1.02621, "step": 187, "tokens/total": 6084304, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 85741} +{"epoch": 0.734375, "grad_norm": 0.3855329751968384, "learning_rate": 7.758374768294647e-05, "loss": 0.007284492254257202, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00731, "step": 188, "tokens/total": 6117896, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 86176} +{"epoch": 0.73828125, "grad_norm": 0.7874730229377747, "learning_rate": 7.733219291524489e-05, "loss": 0.00833985023200512, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00837, "step": 189, "tokens/total": 6152296, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 86649} +{"epoch": 0.7421875, "grad_norm": 0.7271646857261658, "learning_rate": 7.707970881383977e-05, "loss": 0.002648244146257639, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00265, "step": 190, "tokens/total": 6186056, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 87084} +{"epoch": 0.74609375, "grad_norm": 0.37780892848968506, "learning_rate": 7.682630588562518e-05, "loss": 0.00759050203487277, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00762, "step": 191, "tokens/total": 6219160, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 87499} +{"epoch": 0.75, "grad_norm": 0.036821652203798294, "learning_rate": 7.657199467573129e-05, "loss": 0.0006607924588024616, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00066, "step": 192, "tokens/total": 6254840, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 87987} +{"epoch": 0.75390625, "grad_norm": 0.19400620460510254, "learning_rate": 7.631678576708561e-05, "loss": 0.0029009832069277763, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00291, "step": 193, "tokens/total": 6289656, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 88470} +{"epoch": 0.7578125, "grad_norm": 0.45645880699157715, "learning_rate": 7.606068977997255e-05, "loss": 0.0033593974076211452, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00337, "step": 194, "tokens/total": 6321776, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 88923} +{"epoch": 0.76171875, "grad_norm": 0.6249262690544128, "learning_rate": 7.580371737159148e-05, "loss": 0.002935388358309865, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00294, "step": 195, "tokens/total": 6352824, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 89338} +{"epoch": 0.765625, "grad_norm": 0.544479489326477, "learning_rate": 7.554587923561324e-05, "loss": 0.005024977959692478, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00504, "step": 196, "tokens/total": 6388496, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 89812} +{"epoch": 0.76953125, "grad_norm": 0.32891032099723816, "learning_rate": 7.528718610173511e-05, "loss": 0.007746134884655476, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00778, "step": 197, "tokens/total": 6420600, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 90252} +{"epoch": 0.7734375, "grad_norm": 0.384436696767807, "learning_rate": 7.502764873523431e-05, "loss": 0.0037588661070913076, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00377, "step": 198, "tokens/total": 6454616, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 90718} +{"epoch": 0.77734375, "grad_norm": 2.134796142578125, "learning_rate": 7.476727793652011e-05, "loss": 0.014052041806280613, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.01415, "step": 199, "tokens/total": 6485680, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 91194} +{"epoch": 0.78125, "grad_norm": 0.44579821825027466, "learning_rate": 7.450608454068415e-05, "loss": 0.0054313684813678265, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00545, "step": 200, "tokens/total": 6518512, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 91649} +{"epoch": 0.78515625, "grad_norm": 0.21952494978904724, "learning_rate": 7.424407941704987e-05, "loss": 0.0020349775440990925, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.00204, "step": 201, "tokens/total": 6548728, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 92078} +{"epoch": 0.7890625, "grad_norm": 0.9809138774871826, "learning_rate": 7.398127346871986e-05, "loss": 0.007267932407557964, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00729, "step": 202, "tokens/total": 6582192, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 92536} +{"epoch": 0.79296875, "grad_norm": 0.062083899974823, "learning_rate": 7.371767763212238e-05, "loss": 0.0009226897964254022, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.00092, "step": 203, "tokens/total": 6617392, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 93032} +{"epoch": 0.796875, "grad_norm": 0.01781357079744339, "learning_rate": 7.345330287655617e-05, "loss": 0.0002557289262767881, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00026, "step": 204, "tokens/total": 6648904, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 93504} +{"epoch": 0.80078125, "grad_norm": 0.5160856246948242, "learning_rate": 7.31881602037339e-05, "loss": 0.0022948873229324818, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.0023, "step": 205, "tokens/total": 6679832, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 93925} +{"epoch": 0.8046875, "grad_norm": 0.1367982178926468, "learning_rate": 7.29222606473245e-05, "loss": 0.0009511320386081934, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00095, "step": 206, "tokens/total": 6710784, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 94365} +{"epoch": 0.80859375, "grad_norm": 0.04175678268074989, "learning_rate": 7.265561527249383e-05, "loss": 0.000437077134847641, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00044, "step": 207, "tokens/total": 6744848, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 94845} +{"epoch": 0.8125, "grad_norm": 0.45061546564102173, "learning_rate": 7.238823517544436e-05, "loss": 0.0031423845794051886, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00315, "step": 208, "tokens/total": 6777600, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 95285} +{"epoch": 0.81640625, "grad_norm": 0.03075774386525154, "learning_rate": 7.212013148295333e-05, "loss": 0.0004314698453526944, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00043, "step": 209, "tokens/total": 6811024, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 95758} +{"epoch": 0.8203125, "grad_norm": 0.17702266573905945, "learning_rate": 7.185131535190975e-05, "loss": 0.0015619349433109164, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.15, "memory/max_allocated (GiB)": 66.15, "ppl": 1.00156, "step": 210, "tokens/total": 6845376, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 96192} +{"epoch": 0.82421875, "grad_norm": 1.3931230306625366, "learning_rate": 7.158179796885005e-05, "loss": 0.011151259765028954, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01121, "step": 211, "tokens/total": 6880776, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 96641} +{"epoch": 0.828125, "grad_norm": 1.3116214275360107, "learning_rate": 7.131159054949273e-05, "loss": 0.007860398851335049, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00789, "step": 212, "tokens/total": 6916160, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 97160} +{"epoch": 0.83203125, "grad_norm": 0.610227644443512, "learning_rate": 7.104070433827139e-05, "loss": 0.0046451762318611145, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.54, "memory/max_allocated (GiB)": 63.54, "ppl": 1.00466, "step": 213, "tokens/total": 6945320, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 97611} +{"epoch": 0.8359375, "grad_norm": 0.7535299062728882, "learning_rate": 7.076915060786705e-05, "loss": 0.01052122376859188, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.01058, "step": 214, "tokens/total": 6977856, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 98111} +{"epoch": 0.83984375, "grad_norm": 0.07568653672933578, "learning_rate": 7.049694065873882e-05, "loss": 0.0008070214535109699, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.00081, "step": 215, "tokens/total": 7011096, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 98575} +{"epoch": 0.84375, "grad_norm": 1.0808054208755493, "learning_rate": 7.022408581865382e-05, "loss": 0.0063496604561805725, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.93, "memory/max_allocated (GiB)": 63.93, "ppl": 1.00637, "step": 216, "tokens/total": 7041408, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 98991} +{"epoch": 0.84765625, "grad_norm": 0.9029586315155029, "learning_rate": 6.99505974422157e-05, "loss": 0.0077086868695914745, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.6, "memory/max_allocated (GiB)": 63.6, "ppl": 1.00774, "step": 217, "tokens/total": 7071560, "tokens/train_per_sec_per_gpu": 11.31, "tokens/trainable": 99454} +{"epoch": 0.8515625, "grad_norm": 0.609131395816803, "learning_rate": 6.967648691039213e-05, "loss": 0.0015055855037644506, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00151, "step": 218, "tokens/total": 7104696, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 99925} +{"epoch": 0.85546875, "grad_norm": 0.02699223905801773, "learning_rate": 6.940176563004123e-05, "loss": 0.00014496171206701547, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00014, "step": 219, "tokens/total": 7138504, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 100371} +{"epoch": 0.859375, "grad_norm": 0.6436386704444885, "learning_rate": 6.912644503343682e-05, "loss": 0.0059511433355510235, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00597, "step": 220, "tokens/total": 7170992, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 100855} +{"epoch": 0.86328125, "grad_norm": 1.3570032119750977, "learning_rate": 6.885053657779273e-05, "loss": 0.01805398054420948, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.01822, "step": 221, "tokens/total": 7201176, "tokens/train_per_sec_per_gpu": 6.75, "tokens/trainable": 101253} +{"epoch": 0.8671875, "grad_norm": 0.07018955796957016, "learning_rate": 6.857405174478604e-05, "loss": 0.001118581392802298, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00112, "step": 222, "tokens/total": 7233528, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 101706} +{"epoch": 0.87109375, "grad_norm": 1.0221080780029297, "learning_rate": 6.82970020400792e-05, "loss": 0.004950592294335365, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00496, "step": 223, "tokens/total": 7267672, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 102158} +{"epoch": 0.875, "grad_norm": 0.14927849173545837, "learning_rate": 6.801939899284132e-05, "loss": 0.0009229626157321036, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 62.86, "memory/max_allocated (GiB)": 62.86, "ppl": 1.00092, "step": 224, "tokens/total": 7295592, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 102580} +{"epoch": 0.87890625, "grad_norm": 0.6580718755722046, "learning_rate": 6.774125415526827e-05, "loss": 0.011473538354039192, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.01154, "step": 225, "tokens/total": 7325816, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 103063} +{"epoch": 0.8828125, "grad_norm": 0.1309727132320404, "learning_rate": 6.746257910210214e-05, "loss": 0.001691140467301011, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00169, "step": 226, "tokens/total": 7360888, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 103510} +{"epoch": 0.88671875, "grad_norm": 0.1267404854297638, "learning_rate": 6.718338543014937e-05, "loss": 0.0010572320315986872, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00106, "step": 227, "tokens/total": 7390416, "tokens/train_per_sec_per_gpu": 10.11, "tokens/trainable": 103980} +{"epoch": 0.890625, "grad_norm": 1.565649390220642, "learning_rate": 6.69036847577983e-05, "loss": 0.0018468281487002969, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00185, "step": 228, "tokens/total": 7420480, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 104413} +{"epoch": 0.89453125, "grad_norm": 0.0880977213382721, "learning_rate": 6.662348872453553e-05, "loss": 0.0003976537846028805, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.0004, "step": 229, "tokens/total": 7448992, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 104845} +{"epoch": 0.8984375, "grad_norm": 0.07662726193666458, "learning_rate": 6.63428089904618e-05, "loss": 0.0011023804545402527, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.0011, "step": 230, "tokens/total": 7481744, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 105289} +{"epoch": 0.90234375, "grad_norm": 0.199641615152359, "learning_rate": 6.60616572358065e-05, "loss": 0.0036602304317057133, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00367, "step": 231, "tokens/total": 7516704, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 105722} +{"epoch": 0.90625, "grad_norm": 0.08947839587926865, "learning_rate": 6.578004516044172e-05, "loss": 0.0008014455670490861, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.19, "memory/max_allocated (GiB)": 64.19, "ppl": 1.0008, "step": 232, "tokens/total": 7548592, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 106213} +{"epoch": 0.91015625, "grad_norm": 0.00892998743802309, "learning_rate": 6.549798448339548e-05, "loss": 9.443618182558566e-05, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00009, "step": 233, "tokens/total": 7579288, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 106643} +{"epoch": 0.9140625, "grad_norm": 0.33644115924835205, "learning_rate": 6.521548694236384e-05, "loss": 0.0063533238135278225, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00637, "step": 234, "tokens/total": 7611672, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 107136} +{"epoch": 0.91796875, "grad_norm": 0.8189893960952759, "learning_rate": 6.493256429322259e-05, "loss": 0.0023212707601487637, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00232, "step": 235, "tokens/total": 7646288, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 107588} +{"epoch": 0.921875, "grad_norm": 0.29229146242141724, "learning_rate": 6.464922830953799e-05, "loss": 0.0028390558436512947, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00284, "step": 236, "tokens/total": 7680136, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 108087} +{"epoch": 0.92578125, "grad_norm": 1.1160138845443726, "learning_rate": 6.436549078207688e-05, "loss": 0.007048583589494228, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.61, "memory/max_allocated (GiB)": 64.61, "ppl": 1.00707, "step": 237, "tokens/total": 7713072, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 108533} +{"epoch": 0.9296875, "grad_norm": 0.02463982068002224, "learning_rate": 6.408136351831592e-05, "loss": 0.00021469249622896314, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00021, "step": 238, "tokens/total": 7747856, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 108965} +{"epoch": 0.93359375, "grad_norm": 0.5698772072792053, "learning_rate": 6.379685834195036e-05, "loss": 0.009133103303611279, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00917, "step": 239, "tokens/total": 7780136, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 109416} +{"epoch": 0.9375, "grad_norm": 0.3035227656364441, "learning_rate": 6.351198709240186e-05, "loss": 0.0021791686303913593, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.78, "memory/max_allocated (GiB)": 65.78, "ppl": 1.00218, "step": 240, "tokens/total": 7816344, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 109864} +{"epoch": 0.94140625, "grad_norm": 1.034348726272583, "learning_rate": 6.32267616243259e-05, "loss": 0.004682561382651329, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00469, "step": 241, "tokens/total": 7850408, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 110345} +{"epoch": 0.9453125, "grad_norm": 0.10357052087783813, "learning_rate": 6.294119380711849e-05, "loss": 0.0014357276959344745, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00144, "step": 242, "tokens/total": 7881528, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 110794} +{"epoch": 0.94921875, "grad_norm": 0.11051946133375168, "learning_rate": 6.265529552442209e-05, "loss": 0.00030938946292735636, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00031, "step": 243, "tokens/total": 7915712, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 111250} +{"epoch": 0.953125, "grad_norm": 0.45588892698287964, "learning_rate": 6.236907867363127e-05, "loss": 0.002410395536571741, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.61, "memory/max_allocated (GiB)": 64.61, "ppl": 1.00241, "step": 244, "tokens/total": 7947408, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 111693} +{"epoch": 0.95703125, "grad_norm": 0.9705120921134949, "learning_rate": 6.208255516539749e-05, "loss": 0.0041518434882164, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.26, "memory/max_allocated (GiB)": 65.26, "ppl": 1.00416, "step": 245, "tokens/total": 7979600, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 112158} +{"epoch": 0.9609375, "grad_norm": 1.2616232633590698, "learning_rate": 6.179573692313344e-05, "loss": 0.01568039320409298, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.0158, "step": 246, "tokens/total": 8010848, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 112601} +{"epoch": 0.96484375, "grad_norm": 0.04425204172730446, "learning_rate": 6.150863588251694e-05, "loss": 0.0004971876624040306, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.0005, "step": 247, "tokens/total": 8040632, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 113016} +{"epoch": 0.96875, "grad_norm": 0.2645319998264313, "learning_rate": 6.122126399099419e-05, "loss": 0.0004027024260722101, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.0004, "step": 248, "tokens/total": 8072464, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 113476} +{"epoch": 0.97265625, "grad_norm": 0.20700857043266296, "learning_rate": 6.0933633207282615e-05, "loss": 0.0018728254362940788, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.96, "memory/max_allocated (GiB)": 64.96, "ppl": 1.00187, "step": 249, "tokens/total": 8106568, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 113941} +{"epoch": 0.9765625, "grad_norm": 0.39851078391075134, "learning_rate": 6.064575550087316e-05, "loss": 0.004651567433029413, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00466, "step": 250, "tokens/total": 8138496, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 114434} +{"epoch": 0.98046875, "grad_norm": 0.4326707720756531, "learning_rate": 6.0357642851532245e-05, "loss": 0.0019230296602472663, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00192, "step": 251, "tokens/total": 8168080, "tokens/train_per_sec_per_gpu": 10.2, "tokens/trainable": 114898} +{"epoch": 0.984375, "grad_norm": 0.004019869491457939, "learning_rate": 6.0069307248803294e-05, "loss": 2.6091838662978262e-05, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00003, "step": 252, "tokens/total": 8200888, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 115331} +{"epoch": 0.98828125, "grad_norm": 0.02199496701359749, "learning_rate": 5.9780760691507635e-05, "loss": 0.00010118947102455422, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.0001, "step": 253, "tokens/total": 8230704, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 115732} +{"epoch": 0.9921875, "grad_norm": 0.19081470370292664, "learning_rate": 5.9492015187245334e-05, "loss": 0.0039293477311730385, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00394, "step": 254, "tokens/total": 8266560, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 116221} +{"epoch": 0.99609375, "grad_norm": 0.025133121758699417, "learning_rate": 5.920308275189541e-05, "loss": 0.0003061540483031422, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.19, "memory/max_allocated (GiB)": 65.19, "ppl": 1.00031, "step": 255, "tokens/total": 8300184, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 116684} +{"epoch": 1.0, "grad_norm": 0.0504140630364418, "learning_rate": 5.8913975409115874e-05, "loss": 0.00029355520382523537, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00029, "step": 256, "tokens/total": 8333624, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 117126} +{"epoch": 1.00390625, "grad_norm": 0.19541923701763153, "learning_rate": 5.8624705189843395e-05, "loss": 0.0008423158433288336, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00084, "step": 257, "tokens/total": 8364976, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 117592} +{"epoch": 1.0078125, "grad_norm": 0.12562470138072968, "learning_rate": 5.833528413179249e-05, "loss": 0.00022950123820919544, "memory/device_reserved (GiB)": 66.62, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00023, "step": 258, "tokens/total": 8399568, "tokens/train_per_sec_per_gpu": 6.95, "tokens/trainable": 118030} +{"epoch": 1.01171875, "grad_norm": 2.479912757873535, "learning_rate": 5.80457242789548e-05, "loss": 0.01145156379789114, "memory/device_reserved (GiB)": 66.62, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.01152, "step": 259, "tokens/total": 8430616, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 118470} +{"epoch": 1.015625, "grad_norm": 0.12947431206703186, "learning_rate": 5.77560376810977e-05, "loss": 0.0014035464264452457, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.0014, "step": 260, "tokens/total": 8462696, "tokens/train_per_sec_per_gpu": 10.01, "tokens/trainable": 118935} +{"epoch": 1.01953125, "grad_norm": 0.6029449701309204, "learning_rate": 5.7466236393263005e-05, "loss": 0.004066381137818098, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00407, "step": 261, "tokens/total": 8496152, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 119381} +{"epoch": 1.0234375, "grad_norm": 0.2955232858657837, "learning_rate": 5.717633247526522e-05, "loss": 0.000461330491816625, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00046, "step": 262, "tokens/total": 8532912, "tokens/train_per_sec_per_gpu": 7.04, "tokens/trainable": 119849} +{"epoch": 1.02734375, "grad_norm": 0.06511843204498291, "learning_rate": 5.688633799118971e-05, "loss": 0.00023858266649767756, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.00024, "step": 263, "tokens/total": 8565376, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 120319} +{"epoch": 1.03125, "grad_norm": 2.248164653778076, "learning_rate": 5.659626500889066e-05, "loss": 0.004677777178585529, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00469, "step": 264, "tokens/total": 8597640, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 120804} +{"epoch": 1.03515625, "grad_norm": 0.03896769508719444, "learning_rate": 5.6306125599488905e-05, "loss": 9.917314309859648e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.0001, "step": 265, "tokens/total": 8627312, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 121233} +{"epoch": 1.0390625, "grad_norm": 3.0537545680999756, "learning_rate": 5.601593183686955e-05, "loss": 0.0019799652509391308, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.87, "memory/max_allocated (GiB)": 64.87, "ppl": 1.00198, "step": 266, "tokens/total": 8659208, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 121671} +{"epoch": 1.04296875, "grad_norm": 1.564963459968567, "learning_rate": 5.572569579717961e-05, "loss": 0.0072036683559417725, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.00723, "step": 267, "tokens/total": 8693088, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 122141} +{"epoch": 1.046875, "grad_norm": 0.4062555432319641, "learning_rate": 5.543542955832538e-05, "loss": 0.0035484100226312876, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00355, "step": 268, "tokens/total": 8725576, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 122556} +{"epoch": 1.05078125, "grad_norm": 6.586901664733887, "learning_rate": 5.514514519946986e-05, "loss": 0.008012425154447556, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00804, "step": 269, "tokens/total": 8755192, "tokens/train_per_sec_per_gpu": 10.29, "tokens/trainable": 122988} +{"epoch": 1.0546875, "grad_norm": 0.012077665887773037, "learning_rate": 5.485485480053015e-05, "loss": 4.015289596281946e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00004, "step": 270, "tokens/total": 8787704, "tokens/train_per_sec_per_gpu": 10.51, "tokens/trainable": 123478} +{"epoch": 1.05859375, "grad_norm": 0.5642759203910828, "learning_rate": 5.4564570441674645e-05, "loss": 0.004432464484125376, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00444, "step": 271, "tokens/total": 8819768, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 123932} +{"epoch": 1.0625, "grad_norm": 0.03304187208414078, "learning_rate": 5.42743042028204e-05, "loss": 0.00017156133253592998, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00017, "step": 272, "tokens/total": 8853352, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 124428} +{"epoch": 1.06640625, "grad_norm": 0.12834841012954712, "learning_rate": 5.3984068163130464e-05, "loss": 0.000874352001119405, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00087, "step": 273, "tokens/total": 8886840, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 124866} +{"epoch": 1.0703125, "grad_norm": 0.2117948830127716, "learning_rate": 5.369387440051111e-05, "loss": 0.001982636982575059, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00198, "step": 274, "tokens/total": 8920808, "tokens/train_per_sec_per_gpu": 7.09, "tokens/trainable": 125324} +{"epoch": 1.07421875, "grad_norm": 0.04393479973077774, "learning_rate": 5.340373499110935e-05, "loss": 0.0003110459365416318, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00031, "step": 275, "tokens/total": 8955880, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 125813} +{"epoch": 1.078125, "grad_norm": 0.45835569500923157, "learning_rate": 5.3113662008810304e-05, "loss": 0.0019869066309183836, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.00199, "step": 276, "tokens/total": 8989520, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 126252} +{"epoch": 1.08203125, "grad_norm": 0.19344791769981384, "learning_rate": 5.282366752473479e-05, "loss": 0.00041083223186433315, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00041, "step": 277, "tokens/total": 9024680, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 126696} +{"epoch": 1.0859375, "grad_norm": 0.037281036376953125, "learning_rate": 5.2533763606737005e-05, "loss": 0.00030370871536433697, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.0003, "step": 278, "tokens/total": 9056208, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 127173} +{"epoch": 1.08984375, "grad_norm": 0.39025259017944336, "learning_rate": 5.224396231890232e-05, "loss": 0.0019427750958129764, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00194, "step": 279, "tokens/total": 9086376, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 127637} +{"epoch": 1.09375, "grad_norm": 0.2397889941930771, "learning_rate": 5.195427572104522e-05, "loss": 0.0008514882065355778, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00085, "step": 280, "tokens/total": 9118944, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 128112} +{"epoch": 1.09765625, "grad_norm": 0.004315143916755915, "learning_rate": 5.166471586820751e-05, "loss": 1.8962380636367016e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00002, "step": 281, "tokens/total": 9152920, "tokens/train_per_sec_per_gpu": 7.0, "tokens/trainable": 128584} +{"epoch": 1.1015625, "grad_norm": 0.014670480042696, "learning_rate": 5.1375294810156615e-05, "loss": 8.058113598963246e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00008, "step": 282, "tokens/total": 9185728, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 129051} +{"epoch": 1.10546875, "grad_norm": 0.0009366869344376028, "learning_rate": 5.1086024590884144e-05, "loss": 1.2061776942573488e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00001, "step": 283, "tokens/total": 9217984, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 129494} +{"epoch": 1.109375, "grad_norm": 0.28413552045822144, "learning_rate": 5.079691724810461e-05, "loss": 0.000595785619225353, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.0006, "step": 284, "tokens/total": 9253160, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 129983} +{"epoch": 1.11328125, "grad_norm": 0.8271433711051941, "learning_rate": 5.0507984812754684e-05, "loss": 0.007715372834354639, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00775, "step": 285, "tokens/total": 9286136, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 130405} +{"epoch": 1.1171875, "grad_norm": 0.33265364170074463, "learning_rate": 5.021923930849237e-05, "loss": 0.0005701088812202215, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00057, "step": 286, "tokens/total": 9318360, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 130829} +{"epoch": 1.12109375, "grad_norm": 0.09226222336292267, "learning_rate": 4.99306927511967e-05, "loss": 0.00039336696499958634, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00039, "step": 287, "tokens/total": 9350288, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 131287} +{"epoch": 1.125, "grad_norm": 0.752890408039093, "learning_rate": 4.964235714846775e-05, "loss": 0.0009513214463368058, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00095, "step": 288, "tokens/total": 9382104, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 131712} +{"epoch": 1.12890625, "grad_norm": 0.007415190804749727, "learning_rate": 4.9354244499126866e-05, "loss": 4.33319692092482e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00004, "step": 289, "tokens/total": 9414392, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 132168} +{"epoch": 1.1328125, "grad_norm": 0.6453191637992859, "learning_rate": 4.90663667927174e-05, "loss": 0.007516596931964159, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00754, "step": 290, "tokens/total": 9447112, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 132626} +{"epoch": 1.13671875, "grad_norm": 0.0048227072693407536, "learning_rate": 4.877873600900581e-05, "loss": 2.2765148969483562e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00002, "step": 291, "tokens/total": 9483400, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 133119} +{"epoch": 1.140625, "grad_norm": 0.11886513233184814, "learning_rate": 4.849136411748306e-05, "loss": 0.0004103388637304306, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.48, "memory/max_allocated (GiB)": 64.48, "ppl": 1.00041, "step": 292, "tokens/total": 9513896, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 133550} +{"epoch": 1.14453125, "grad_norm": 0.026430781930685043, "learning_rate": 4.8204263076866574e-05, "loss": 0.00010555988410487771, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00011, "step": 293, "tokens/total": 9550184, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 133992} +{"epoch": 1.1484375, "grad_norm": 0.012441251426935196, "learning_rate": 4.791744483460251e-05, "loss": 0.00011833473399747163, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00012, "step": 294, "tokens/total": 9583920, "tokens/train_per_sec_per_gpu": 10.07, "tokens/trainable": 134467} +{"epoch": 1.15234375, "grad_norm": 0.25599414110183716, "learning_rate": 4.7630921326368736e-05, "loss": 0.0006650862051174045, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00067, "step": 295, "tokens/total": 9617288, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 134895} +{"epoch": 1.15625, "grad_norm": 0.10321811586618423, "learning_rate": 4.7344704475577916e-05, "loss": 0.0003347288293298334, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 62.94, "memory/max_allocated (GiB)": 62.94, "ppl": 1.00033, "step": 296, "tokens/total": 9645600, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 135321} +{"epoch": 1.16015625, "grad_norm": 0.44273266196250916, "learning_rate": 4.705880619288153e-05, "loss": 0.0010826209327206016, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00108, "step": 297, "tokens/total": 9679456, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 135765} +{"epoch": 1.1640625, "grad_norm": 0.010997132398188114, "learning_rate": 4.677323837567412e-05, "loss": 2.2905387595528737e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00002, "step": 298, "tokens/total": 9711696, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 136196} +{"epoch": 1.16796875, "grad_norm": 0.014206001535058022, "learning_rate": 4.6488012907598146e-05, "loss": 9.898875578073785e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.0001, "step": 299, "tokens/total": 9745792, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 136649} +{"epoch": 1.171875, "grad_norm": 1.1384004354476929, "learning_rate": 4.620314165804964e-05, "loss": 0.014785472303628922, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.0149, "step": 300, "tokens/total": 9780648, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 137128} +{"epoch": 1.17578125, "grad_norm": 0.5795979499816895, "learning_rate": 4.591863648168407e-05, "loss": 0.007560649886727333, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00759, "step": 301, "tokens/total": 9813032, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 137585} +{"epoch": 1.1796875, "grad_norm": 0.20474053919315338, "learning_rate": 4.5634509217923135e-05, "loss": 0.001349215628579259, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00135, "step": 302, "tokens/total": 9845200, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 138054} +{"epoch": 1.18359375, "grad_norm": 0.3589054346084595, "learning_rate": 4.535077169046201e-05, "loss": 0.0013046972453594208, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.29, "memory/max_allocated (GiB)": 65.29, "ppl": 1.00131, "step": 303, "tokens/total": 9878616, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 138559} +{"epoch": 1.1875, "grad_norm": 0.44081878662109375, "learning_rate": 4.506743570677743e-05, "loss": 0.019181642681360245, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.01937, "step": 304, "tokens/total": 9910776, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 139050} +{"epoch": 1.19140625, "grad_norm": 0.6967405080795288, "learning_rate": 4.478451305763618e-05, "loss": 0.006568855606019497, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00659, "step": 305, "tokens/total": 9945480, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 139525} +{"epoch": 1.1953125, "grad_norm": 0.03732616454362869, "learning_rate": 4.450201551660454e-05, "loss": 0.00018702908710110933, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.00019, "step": 306, "tokens/total": 9975992, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 139969} +{"epoch": 1.19921875, "grad_norm": 1.4307674169540405, "learning_rate": 4.4219954839558276e-05, "loss": 0.002400221535935998, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.0024, "step": 307, "tokens/total": 10006440, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 140443} +{"epoch": 1.203125, "grad_norm": 0.0062005710788071156, "learning_rate": 4.393834276419352e-05, "loss": 7.27250735508278e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00007, "step": 308, "tokens/total": 10042928, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 140926} +{"epoch": 1.20703125, "grad_norm": 2.3836653232574463, "learning_rate": 4.36571910095382e-05, "loss": 0.006360674276947975, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00638, "step": 309, "tokens/total": 10075184, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 141344} +{"epoch": 1.2109375, "grad_norm": 0.3484824299812317, "learning_rate": 4.337651127546448e-05, "loss": 0.006263189949095249, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00628, "step": 310, "tokens/total": 10106656, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 141753} +{"epoch": 1.21484375, "grad_norm": 0.597235381603241, "learning_rate": 4.3096315242201736e-05, "loss": 0.005300628952682018, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.82, "memory/max_allocated (GiB)": 63.82, "ppl": 1.00531, "step": 311, "tokens/total": 10138536, "tokens/train_per_sec_per_gpu": 10.04, "tokens/trainable": 142219} +{"epoch": 1.21875, "grad_norm": 0.2574526369571686, "learning_rate": 4.2816614569850635e-05, "loss": 0.002176616806536913, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.00218, "step": 312, "tokens/total": 10165768, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 142617} +{"epoch": 1.22265625, "grad_norm": 0.07274571061134338, "learning_rate": 4.2537420897897864e-05, "loss": 0.00072545709554106, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00073, "step": 313, "tokens/total": 10201624, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 143103} +{"epoch": 1.2265625, "grad_norm": 0.06118059903383255, "learning_rate": 4.225874584473174e-05, "loss": 0.0006570147234015167, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00066, "step": 314, "tokens/total": 10233304, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 143535} +{"epoch": 1.23046875, "grad_norm": 0.022029735147953033, "learning_rate": 4.19806010071587e-05, "loss": 0.00017686569481156766, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00018, "step": 315, "tokens/total": 10267904, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 144009} +{"epoch": 1.234375, "grad_norm": 0.6411435604095459, "learning_rate": 4.170299795992081e-05, "loss": 0.008778381161391735, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00882, "step": 316, "tokens/total": 10301280, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 144460} +{"epoch": 1.23828125, "grad_norm": 0.24405646324157715, "learning_rate": 4.142594825521398e-05, "loss": 0.0051880949176847935, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.0052, "step": 317, "tokens/total": 10337072, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 144936} +{"epoch": 1.2421875, "grad_norm": 0.03994289040565491, "learning_rate": 4.114946342220728e-05, "loss": 0.00017215024854522198, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.69, "memory/max_allocated (GiB)": 64.69, "ppl": 1.00017, "step": 318, "tokens/total": 10368688, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 145375} +{"epoch": 1.24609375, "grad_norm": 0.0403224416077137, "learning_rate": 4.087355496656321e-05, "loss": 0.0006472209352068603, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00065, "step": 319, "tokens/total": 10400752, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 145849} +{"epoch": 1.25, "grad_norm": 1.2951502799987793, "learning_rate": 4.05982343699588e-05, "loss": 0.0015427838079631329, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00154, "step": 320, "tokens/total": 10432112, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 146318} +{"epoch": 1.25390625, "grad_norm": 0.1094268187880516, "learning_rate": 4.0323513089607876e-05, "loss": 0.0006199459894560277, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00062, "step": 321, "tokens/total": 10465472, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 146821} +{"epoch": 1.2578125, "grad_norm": 0.24705199897289276, "learning_rate": 4.004940255778431e-05, "loss": 0.0013288812479004264, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00133, "step": 322, "tokens/total": 10496336, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 147285} +{"epoch": 1.26171875, "grad_norm": 0.011125920340418816, "learning_rate": 3.977591418134619e-05, "loss": 0.0001005389349302277, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.0001, "step": 323, "tokens/total": 10525864, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 147729} +{"epoch": 1.265625, "grad_norm": 1.0757675170898438, "learning_rate": 3.95030593412612e-05, "loss": 0.001674158964306116, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00168, "step": 324, "tokens/total": 10559680, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 148169} +{"epoch": 1.26953125, "grad_norm": 0.21791519224643707, "learning_rate": 3.923084939213296e-05, "loss": 0.0013428742531687021, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00134, "step": 325, "tokens/total": 10595400, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 148630} +{"epoch": 1.2734375, "grad_norm": 0.0047524478286504745, "learning_rate": 3.895929566172861e-05, "loss": 6.259949441300705e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00006, "step": 326, "tokens/total": 10626680, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 149063} +{"epoch": 1.27734375, "grad_norm": 0.019786324352025986, "learning_rate": 3.868840945050728e-05, "loss": 0.00012708311260212213, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00013, "step": 327, "tokens/total": 10661784, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 149549} +{"epoch": 1.28125, "grad_norm": 0.1601538211107254, "learning_rate": 3.841820203114995e-05, "loss": 0.004930198658257723, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.31, "memory/max_allocated (GiB)": 65.31, "ppl": 1.00494, "step": 328, "tokens/total": 10696184, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 150015} +{"epoch": 1.28515625, "grad_norm": 0.008667239919304848, "learning_rate": 3.814868464809027e-05, "loss": 5.6575288908788934e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00006, "step": 329, "tokens/total": 10729376, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 150514} +{"epoch": 1.2890625, "grad_norm": 0.2909683287143707, "learning_rate": 3.787986851704667e-05, "loss": 0.000949668581597507, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00095, "step": 330, "tokens/total": 10764824, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 151021} +{"epoch": 1.29296875, "grad_norm": 0.018113266676664352, "learning_rate": 3.7611764824555654e-05, "loss": 0.00024685749667696655, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00025, "step": 331, "tokens/total": 10797072, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 151486} +{"epoch": 1.296875, "grad_norm": 0.2995886504650116, "learning_rate": 3.734438472750619e-05, "loss": 0.002260790206491947, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00226, "step": 332, "tokens/total": 10829376, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 151956} +{"epoch": 1.30078125, "grad_norm": 0.005574777256697416, "learning_rate": 3.707773935267552e-05, "loss": 7.154985360102728e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00007, "step": 333, "tokens/total": 10863512, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 152442} +{"epoch": 1.3046875, "grad_norm": 0.08149824291467667, "learning_rate": 3.68118397962661e-05, "loss": 0.00014040371752344072, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00014, "step": 334, "tokens/total": 10894432, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 152890} +{"epoch": 1.30859375, "grad_norm": 0.7589680552482605, "learning_rate": 3.654669712344384e-05, "loss": 0.008459954522550106, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.34, "memory/max_allocated (GiB)": 63.34, "ppl": 1.0085, "step": 335, "tokens/total": 10924848, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 153368} +{"epoch": 1.3125, "grad_norm": 0.009604155085980892, "learning_rate": 3.628232236787763e-05, "loss": 0.00010219989053439349, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.0001, "step": 336, "tokens/total": 10959024, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 153809} +{"epoch": 1.31640625, "grad_norm": 0.6132693290710449, "learning_rate": 3.6018726531280144e-05, "loss": 0.004038605839014053, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00405, "step": 337, "tokens/total": 10991088, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 154258} +{"epoch": 1.3203125, "grad_norm": 0.026054130867123604, "learning_rate": 3.575592058295017e-05, "loss": 0.00031873409170657396, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00032, "step": 338, "tokens/total": 11023904, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 154704} +{"epoch": 1.32421875, "grad_norm": 0.12394510954618454, "learning_rate": 3.549391545931585e-05, "loss": 0.0012322301045060158, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00123, "step": 339, "tokens/total": 11057680, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 155157} +{"epoch": 1.328125, "grad_norm": 0.09266605228185654, "learning_rate": 3.5232722063479914e-05, "loss": 0.0006048950017429888, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00061, "step": 340, "tokens/total": 11089800, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 155582} +{"epoch": 1.33203125, "grad_norm": 0.014377498999238014, "learning_rate": 3.49723512647657e-05, "loss": 6.094536365708336e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00006, "step": 341, "tokens/total": 11117960, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 156047} +{"epoch": 1.3359375, "grad_norm": 0.24727164208889008, "learning_rate": 3.471281389826491e-05, "loss": 0.004117344040423632, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00413, "step": 342, "tokens/total": 11151888, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 156500} +{"epoch": 1.33984375, "grad_norm": 0.9164970517158508, "learning_rate": 3.4454120764386764e-05, "loss": 0.009965156204998493, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.22, "memory/max_allocated (GiB)": 65.22, "ppl": 1.01001, "step": 343, "tokens/total": 11184568, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 156931} +{"epoch": 1.34375, "grad_norm": 0.44967979192733765, "learning_rate": 3.4196282628408526e-05, "loss": 0.016884606331586838, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.01703, "step": 344, "tokens/total": 11216400, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 157385} +{"epoch": 1.34765625, "grad_norm": 0.019384995102882385, "learning_rate": 3.3939310220027456e-05, "loss": 0.00011040569370379671, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00011, "step": 345, "tokens/total": 11250768, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 157817} +{"epoch": 1.3515625, "grad_norm": 0.007810494396835566, "learning_rate": 3.3683214232914404e-05, "loss": 9.345552098238841e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00009, "step": 346, "tokens/total": 11281808, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 158273} +{"epoch": 1.35546875, "grad_norm": 0.12310615926980972, "learning_rate": 3.342800532426873e-05, "loss": 0.0014010285958647728, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.0014, "step": 347, "tokens/total": 11313968, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 158728} +{"epoch": 1.359375, "grad_norm": 0.0026526886504143476, "learning_rate": 3.317369411437484e-05, "loss": 3.6494049709290266e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00004, "step": 348, "tokens/total": 11347288, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 159168} +{"epoch": 1.36328125, "grad_norm": 0.16576950252056122, "learning_rate": 3.292029118616024e-05, "loss": 0.0018154990393668413, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00182, "step": 349, "tokens/total": 11377128, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 159592} +{"epoch": 1.3671875, "grad_norm": 0.9273552298545837, "learning_rate": 3.266780708475511e-05, "loss": 0.010567938908934593, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.01062, "step": 350, "tokens/total": 11409488, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 160093} +{"epoch": 1.37109375, "grad_norm": 0.1576709896326065, "learning_rate": 3.241625231705354e-05, "loss": 0.0006493964465335011, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00065, "step": 351, "tokens/total": 11442040, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 160534} +{"epoch": 1.375, "grad_norm": 0.07528169453144073, "learning_rate": 3.216563735127618e-05, "loss": 0.0006068511283956468, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00061, "step": 352, "tokens/total": 11478312, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 161009} +{"epoch": 1.37890625, "grad_norm": 0.16244545578956604, "learning_rate": 3.191597261653475e-05, "loss": 0.0025819032453000546, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.93, "memory/max_allocated (GiB)": 63.93, "ppl": 1.00259, "step": 353, "tokens/total": 11509008, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 161450} +{"epoch": 1.3828125, "grad_norm": 0.4314417839050293, "learning_rate": 3.166726850239794e-05, "loss": 0.013334466144442558, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.01342, "step": 354, "tokens/total": 11541264, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 161900} +{"epoch": 1.38671875, "grad_norm": 0.005413959268480539, "learning_rate": 3.141953535845912e-05, "loss": 4.548897413769737e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00005, "step": 355, "tokens/total": 11573072, "tokens/train_per_sec_per_gpu": 10.06, "tokens/trainable": 162358} +{"epoch": 1.390625, "grad_norm": 0.25517818331718445, "learning_rate": 3.11727834939056e-05, "loss": 0.0018605251098051667, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00186, "step": 356, "tokens/total": 11605224, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 162845} +{"epoch": 1.39453125, "grad_norm": 0.15159903466701508, "learning_rate": 3.092702317708967e-05, "loss": 0.0018348200246691704, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00184, "step": 357, "tokens/total": 11637512, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 163299} +{"epoch": 1.3984375, "grad_norm": 0.3781057596206665, "learning_rate": 3.0682264635101276e-05, "loss": 0.004011630546301603, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00402, "step": 358, "tokens/total": 11671280, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 163774} +{"epoch": 1.40234375, "grad_norm": 0.08872517943382263, "learning_rate": 3.0438518053342407e-05, "loss": 0.0008807433187030256, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00088, "step": 359, "tokens/total": 11702008, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 164199} +{"epoch": 1.40625, "grad_norm": 0.245505228638649, "learning_rate": 3.0195793575103266e-05, "loss": 0.001850585569627583, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00185, "step": 360, "tokens/total": 11732032, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 164667} +{"epoch": 1.41015625, "grad_norm": 0.06616068631410599, "learning_rate": 2.9954101301140146e-05, "loss": 0.000672110531013459, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00067, "step": 361, "tokens/total": 11767472, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.13047614693641663, "learning_rate": 2.9713451289255123e-05, "loss": 0.0005617979331873357, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00056, "step": 362, "tokens/total": 11798432, "tokens/train_per_sec_per_gpu": 10.43, "tokens/trainable": 165575} +{"epoch": 1.41796875, "grad_norm": 1.1058882474899292, "learning_rate": 2.9473853553877484e-05, "loss": 0.015576017089188099, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.0157, "step": 363, "tokens/total": 11833144, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 166060} +{"epoch": 1.421875, "grad_norm": 0.01754162460565567, "learning_rate": 2.9235318065647e-05, "loss": 8.866645657690242e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.00009, "step": 364, "tokens/total": 11865856, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 166478} +{"epoch": 1.42578125, "grad_norm": 0.5927127003669739, "learning_rate": 2.8997854750998964e-05, "loss": 0.0044198124669492245, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00443, "step": 365, "tokens/total": 11900400, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 166941} +{"epoch": 1.4296875, "grad_norm": 0.23876310884952545, "learning_rate": 2.8761473491751258e-05, "loss": 0.0023099309764802456, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.49, "memory/max_allocated (GiB)": 63.49, "ppl": 1.00231, "step": 366, "tokens/total": 11931368, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 167381} +{"epoch": 1.43359375, "grad_norm": 0.03671685978770256, "learning_rate": 2.8526184124692883e-05, "loss": 0.0003434178070165217, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00034, "step": 367, "tokens/total": 11963320, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 167830} +{"epoch": 1.4375, "grad_norm": 0.006149331573396921, "learning_rate": 2.829199644117484e-05, "loss": 7.864625513320789e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00008, "step": 368, "tokens/total": 11997536, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 168266} +{"epoch": 1.44140625, "grad_norm": 0.01878887228667736, "learning_rate": 2.8058920186702553e-05, "loss": 0.0001958647626452148, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.0002, "step": 369, "tokens/total": 12029528, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 168716} +{"epoch": 1.4453125, "grad_norm": 0.08692418038845062, "learning_rate": 2.782696506053033e-05, "loss": 0.0008446520660072565, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00085, "step": 370, "tokens/total": 12062504, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 169176} +{"epoch": 1.44921875, "grad_norm": 0.05329706147313118, "learning_rate": 2.7596140715257824e-05, "loss": 0.0006614453159272671, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.78, "memory/max_allocated (GiB)": 65.78, "ppl": 1.00066, "step": 371, "tokens/total": 12097376, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 169591} +{"epoch": 1.453125, "grad_norm": 0.044359080493450165, "learning_rate": 2.7366456756428184e-05, "loss": 0.0006114643765613437, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00061, "step": 372, "tokens/total": 12131920, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 170041} +{"epoch": 1.45703125, "grad_norm": 0.0367194302380085, "learning_rate": 2.7137922742128486e-05, "loss": 0.00040470942622050643, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.0004, "step": 373, "tokens/total": 12164528, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 170516} +{"epoch": 1.4609375, "grad_norm": 0.028809333220124245, "learning_rate": 2.691054818259188e-05, "loss": 0.00034480783506296575, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.00034, "step": 374, "tokens/total": 12197376, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 170952} +{"epoch": 1.46484375, "grad_norm": 0.009195424616336823, "learning_rate": 2.6684342539801933e-05, "loss": 0.00011393396562198177, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00011, "step": 375, "tokens/total": 12233104, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 171424} +{"epoch": 1.46875, "grad_norm": 0.057008057832717896, "learning_rate": 2.645931522709877e-05, "loss": 0.00022495375014841557, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.69, "memory/max_allocated (GiB)": 65.69, "ppl": 1.00022, "step": 376, "tokens/total": 12269224, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 171884} +{"epoch": 1.47265625, "grad_norm": 0.002363579347729683, "learning_rate": 2.6235475608787365e-05, "loss": 4.105380503460765e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00004, "step": 377, "tokens/total": 12302952, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 172404} +{"epoch": 1.4765625, "grad_norm": 0.06930650025606155, "learning_rate": 2.6012832999747916e-05, "loss": 0.0005112551734782755, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00051, "step": 378, "tokens/total": 12338424, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 172871} +{"epoch": 1.48046875, "grad_norm": 0.2741300165653229, "learning_rate": 2.579139666504821e-05, "loss": 0.0025701343547552824, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00257, "step": 379, "tokens/total": 12369144, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 173329} +{"epoch": 1.484375, "grad_norm": 0.052740972489118576, "learning_rate": 2.557117581955798e-05, "loss": 0.00023415681789629161, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00023, "step": 380, "tokens/total": 12403352, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 173827} +{"epoch": 1.48828125, "grad_norm": 0.014583371579647064, "learning_rate": 2.5352179627565532e-05, "loss": 0.00015910080401226878, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00016, "step": 381, "tokens/total": 12441056, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 174287} +{"epoch": 1.4921875, "grad_norm": 0.19080786406993866, "learning_rate": 2.5134417202396277e-05, "loss": 0.0009539278689771891, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.00095, "step": 382, "tokens/total": 12474080, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 174731} +{"epoch": 1.49609375, "grad_norm": 1.1165595054626465, "learning_rate": 2.491789760603361e-05, "loss": 0.004101225174963474, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00411, "step": 383, "tokens/total": 12508992, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 175183} +{"epoch": 1.5, "grad_norm": 0.038611508905887604, "learning_rate": 2.4702629848741764e-05, "loss": 0.00035253437818028033, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00035, "step": 384, "tokens/total": 12540896, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 175655} +{"epoch": 1.50390625, "grad_norm": 1.9860975742340088, "learning_rate": 2.4488622888690785e-05, "loss": 0.02397940494120121, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.02427, "step": 385, "tokens/total": 12568416, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 176085} +{"epoch": 1.5078125, "grad_norm": 0.0032743054907768965, "learning_rate": 2.427588563158384e-05, "loss": 4.679711128119379e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00005, "step": 386, "tokens/total": 12599792, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 176549} +{"epoch": 1.51171875, "grad_norm": 0.06001530960202217, "learning_rate": 2.406442693028651e-05, "loss": 0.0004124731640331447, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00041, "step": 387, "tokens/total": 12631224, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 176995} +{"epoch": 1.515625, "grad_norm": 0.2770403325557709, "learning_rate": 2.3854255584458547e-05, "loss": 0.0013393640983849764, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00134, "step": 388, "tokens/total": 12667240, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 177468} +{"epoch": 1.51953125, "grad_norm": 0.11266841739416122, "learning_rate": 2.3645380340187508e-05, "loss": 0.0012668573763221502, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.54, "memory/max_allocated (GiB)": 65.54, "ppl": 1.00127, "step": 389, "tokens/total": 12700488, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 177961} +{"epoch": 1.5234375, "grad_norm": 0.0023142362479120493, "learning_rate": 2.3437809889624914e-05, "loss": 3.382587237865664e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.00003, "step": 390, "tokens/total": 12734528, "tokens/train_per_sec_per_gpu": 6.94, "tokens/trainable": 178417} +{"epoch": 1.52734375, "grad_norm": 0.0490071065723896, "learning_rate": 2.3231552870624487e-05, "loss": 0.0005428973236121237, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00054, "step": 391, "tokens/total": 12767152, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 178862} +{"epoch": 1.53125, "grad_norm": 0.08045181632041931, "learning_rate": 2.3026617866382657e-05, "loss": 0.0005516082746908069, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.00055, "step": 392, "tokens/total": 12798168, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 179320} +{"epoch": 1.53515625, "grad_norm": 0.01532567199319601, "learning_rate": 2.2823013405081507e-05, "loss": 0.00013095536269247532, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00013, "step": 393, "tokens/total": 12828896, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 179758} +{"epoch": 1.5390625, "grad_norm": 1.0155693292617798, "learning_rate": 2.2620747959533722e-05, "loss": 0.006862460169941187, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00689, "step": 394, "tokens/total": 12860024, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 180211} +{"epoch": 1.54296875, "grad_norm": 0.040735747665166855, "learning_rate": 2.2419829946830123e-05, "loss": 0.0001847826933953911, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00018, "step": 395, "tokens/total": 12891472, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 180637} +{"epoch": 1.546875, "grad_norm": 0.07616914808750153, "learning_rate": 2.2220267727989325e-05, "loss": 0.0007961453520692885, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.0008, "step": 396, "tokens/total": 12925648, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 181142} +{"epoch": 1.55078125, "grad_norm": 0.07733720541000366, "learning_rate": 2.202206960760984e-05, "loss": 0.0004319329163990915, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.77, "memory/max_allocated (GiB)": 64.77, "ppl": 1.00043, "step": 397, "tokens/total": 12959200, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 181597} +{"epoch": 1.5546875, "grad_norm": 0.07428721338510513, "learning_rate": 2.182524383352446e-05, "loss": 0.00044326152419671416, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00044, "step": 398, "tokens/total": 12989624, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 182030} +{"epoch": 1.55859375, "grad_norm": 0.001346423290669918, "learning_rate": 2.1629798596457056e-05, "loss": 2.0588728148140945e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 62.97, "memory/max_allocated (GiB)": 62.97, "ppl": 1.00002, "step": 399, "tokens/total": 13019040, "tokens/train_per_sec_per_gpu": 10.28, "tokens/trainable": 182496} +{"epoch": 1.5625, "grad_norm": 0.26732707023620605, "learning_rate": 2.1435742029681725e-05, "loss": 0.0010752055095508695, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00108, "step": 400, "tokens/total": 13051808, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 182963} +{"epoch": 1.56640625, "grad_norm": 0.41720569133758545, "learning_rate": 2.124308220868431e-05, "loss": 0.005927528720349073, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00595, "step": 401, "tokens/total": 13085520, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 183416} +{"epoch": 1.5703125, "grad_norm": 0.007287142798304558, "learning_rate": 2.105182715082638e-05, "loss": 6.961113103898242e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00007, "step": 402, "tokens/total": 13116224, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 183857} +{"epoch": 1.57421875, "grad_norm": 0.20805464684963226, "learning_rate": 2.0861984815011552e-05, "loss": 0.0007156135397963226, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00072, "step": 403, "tokens/total": 13149984, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 184296} +{"epoch": 1.578125, "grad_norm": 0.2891780138015747, "learning_rate": 2.0673563101354323e-05, "loss": 0.0036118626594543457, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00362, "step": 404, "tokens/total": 13185488, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 184719} +{"epoch": 1.58203125, "grad_norm": 0.16423697769641876, "learning_rate": 2.0486569850851317e-05, "loss": 0.0012904690811410546, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00129, "step": 405, "tokens/total": 13216320, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 185172} +{"epoch": 1.5859375, "grad_norm": 0.042291197925806046, "learning_rate": 2.0301012845054956e-05, "loss": 0.0003011793887708336, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.0003, "step": 406, "tokens/total": 13248136, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 185646} +{"epoch": 1.58984375, "grad_norm": 0.019241247326135635, "learning_rate": 2.011689980574966e-05, "loss": 0.00014123138680588454, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00014, "step": 407, "tokens/total": 13280640, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 186129} +{"epoch": 1.59375, "grad_norm": 0.29163017868995667, "learning_rate": 1.993423839463052e-05, "loss": 0.0014088965253904462, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00141, "step": 408, "tokens/total": 13314160, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 186610} +{"epoch": 1.59765625, "grad_norm": 0.050234850496053696, "learning_rate": 1.975303621298445e-05, "loss": 0.0006015418330207467, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.51, "memory/max_allocated (GiB)": 63.51, "ppl": 1.0006, "step": 409, "tokens/total": 13345024, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 187066} +{"epoch": 1.6015625, "grad_norm": 0.032205116003751755, "learning_rate": 1.957330080137385e-05, "loss": 0.000110056389530655, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00011, "step": 410, "tokens/total": 13377248, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 187567} +{"epoch": 1.60546875, "grad_norm": 0.014533870853483677, "learning_rate": 1.9395039639322864e-05, "loss": 0.00012756846263073385, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00013, "step": 411, "tokens/total": 13410992, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 188056} +{"epoch": 1.609375, "grad_norm": 0.003041240619495511, "learning_rate": 1.9218260145006073e-05, "loss": 3.300994649180211e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00003, "step": 412, "tokens/total": 13442728, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 188502} +{"epoch": 1.61328125, "grad_norm": 0.015377112664282322, "learning_rate": 1.904296967493982e-05, "loss": 0.0001846793165896088, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00018, "step": 413, "tokens/total": 13474824, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 188995} +{"epoch": 1.6171875, "grad_norm": 0.025745412334799767, "learning_rate": 1.8869175523676064e-05, "loss": 0.00026183496811427176, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00026, "step": 414, "tokens/total": 13506528, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 189483} +{"epoch": 1.62109375, "grad_norm": 0.30711475014686584, "learning_rate": 1.869688492349885e-05, "loss": 0.0025940004270523787, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.0026, "step": 415, "tokens/total": 13537792, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 189935} +{"epoch": 1.625, "grad_norm": 0.6189919710159302, "learning_rate": 1.85261050441233e-05, "loss": 0.0035590599291026592, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.00357, "step": 416, "tokens/total": 13573136, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 190399} +{"epoch": 1.62890625, "grad_norm": 0.033013854175806046, "learning_rate": 1.8356842992397304e-05, "loss": 0.00017999968258664012, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00018, "step": 417, "tokens/total": 13607240, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 190862} +{"epoch": 1.6328125, "grad_norm": 0.0008719033212400973, "learning_rate": 1.8189105812005714e-05, "loss": 7.976420420163777e-06, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00001, "step": 418, "tokens/total": 13638920, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 191294} +{"epoch": 1.63671875, "grad_norm": 0.7442092895507812, "learning_rate": 1.802290048317732e-05, "loss": 0.006766983773559332, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00679, "step": 419, "tokens/total": 13672320, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 191764} +{"epoch": 1.640625, "grad_norm": 0.008391626179218292, "learning_rate": 1.785823392239424e-05, "loss": 9.823511936701834e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.0001, "step": 420, "tokens/total": 13703312, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 192196} +{"epoch": 1.64453125, "grad_norm": 0.03226237744092941, "learning_rate": 1.7695112982104225e-05, "loss": 0.00017068168381229043, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00017, "step": 421, "tokens/total": 13734440, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 192612} +{"epoch": 1.6484375, "grad_norm": 0.0006009025382809341, "learning_rate": 1.7533544450435433e-05, "loss": 8.9460863819113e-06, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.62, "memory/max_allocated (GiB)": 64.62, "ppl": 1.00001, "step": 422, "tokens/total": 13766944, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 193059} +{"epoch": 1.65234375, "grad_norm": 0.679267406463623, "learning_rate": 1.7373535050913946e-05, "loss": 0.0077064866200089455, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00774, "step": 423, "tokens/total": 13797120, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 193466} +{"epoch": 1.65625, "grad_norm": 0.007342013996094465, "learning_rate": 1.721509144218405e-05, "loss": 6.789407052565366e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00007, "step": 424, "tokens/total": 13828040, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 193910} +{"epoch": 1.66015625, "grad_norm": 0.00429494958370924, "learning_rate": 1.705822021773101e-05, "loss": 2.7031803256249987e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.06, "memory/max_allocated (GiB)": 63.06, "ppl": 1.00003, "step": 425, "tokens/total": 13856328, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 194314} +{"epoch": 1.6640625, "grad_norm": 0.045290976762771606, "learning_rate": 1.69029279056068e-05, "loss": 0.0004738906864076853, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00047, "step": 426, "tokens/total": 13891312, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 194782} +{"epoch": 1.66796875, "grad_norm": 0.4459809362888336, "learning_rate": 1.6749220968158415e-05, "loss": 0.005655400454998016, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00567, "step": 427, "tokens/total": 13925272, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 195254} +{"epoch": 1.671875, "grad_norm": 0.13300973176956177, "learning_rate": 1.659710580175893e-05, "loss": 0.0007448334363289177, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.53, "memory/max_allocated (GiB)": 63.53, "ppl": 1.00075, "step": 428, "tokens/total": 13955752, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 195702} +{"epoch": 1.67578125, "grad_norm": 0.15064287185668945, "learning_rate": 1.644658873654133e-05, "loss": 0.001153715536929667, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00115, "step": 429, "tokens/total": 13988000, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 196164} +{"epoch": 1.6796875, "grad_norm": 0.06837531179189682, "learning_rate": 1.629767603613508e-05, "loss": 0.0005676149157807231, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00057, "step": 430, "tokens/total": 14018712, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 196604} +{"epoch": 1.68359375, "grad_norm": 1.1016595363616943, "learning_rate": 1.615037389740547e-05, "loss": 0.001525315223261714, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00153, "step": 431, "tokens/total": 14051992, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 197070} +{"epoch": 1.6875, "grad_norm": 0.07849308103322983, "learning_rate": 1.600468845019576e-05, "loss": 0.00021543535694945604, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.36, "memory/max_allocated (GiB)": 63.36, "ppl": 1.00022, "step": 432, "tokens/total": 14082624, "tokens/train_per_sec_per_gpu": 10.35, "tokens/trainable": 197552} +{"epoch": 1.69140625, "grad_norm": 0.021648166701197624, "learning_rate": 1.5860625757072092e-05, "loss": 0.0001516887277830392, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00015, "step": 433, "tokens/total": 14113304, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 197990} +{"epoch": 1.6953125, "grad_norm": 0.0803137719631195, "learning_rate": 1.571819181307116e-05, "loss": 0.0006194700836203992, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00062, "step": 434, "tokens/total": 14149592, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 198513} +{"epoch": 1.69921875, "grad_norm": 0.092012919485569, "learning_rate": 1.557739254545075e-05, "loss": 0.0005670435493811965, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00057, "step": 435, "tokens/total": 14183584, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 198979} +{"epoch": 1.703125, "grad_norm": 0.09311285614967346, "learning_rate": 1.543823381344311e-05, "loss": 0.000716895330697298, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00072, "step": 436, "tokens/total": 14213688, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 199443} +{"epoch": 1.70703125, "grad_norm": 0.02055373415350914, "learning_rate": 1.5300721408011114e-05, "loss": 0.00014890942838974297, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.63, "memory/max_allocated (GiB)": 65.63, "ppl": 1.00015, "step": 437, "tokens/total": 14246480, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 199899} +{"epoch": 1.7109375, "grad_norm": 0.05361669883131981, "learning_rate": 1.5164861051607254e-05, "loss": 0.00020509117166511714, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00021, "step": 438, "tokens/total": 14278320, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 200368} +{"epoch": 1.71484375, "grad_norm": 0.5552303194999695, "learning_rate": 1.5030658397935521e-05, "loss": 0.00504662049934268, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00506, "step": 439, "tokens/total": 14313016, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 200819} +{"epoch": 1.71875, "grad_norm": 0.15168561041355133, "learning_rate": 1.4898119031716104e-05, "loss": 0.0013616083888337016, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00136, "step": 440, "tokens/total": 14343352, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 201275} +{"epoch": 1.72265625, "grad_norm": 0.008403181098401546, "learning_rate": 1.476724846845306e-05, "loss": 4.901175998384133e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00005, "step": 441, "tokens/total": 14377736, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 201699} +{"epoch": 1.7265625, "grad_norm": 0.07509441673755646, "learning_rate": 1.463805215420471e-05, "loss": 0.0009472158853895962, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00095, "step": 442, "tokens/total": 14415056, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 202200} +{"epoch": 1.73046875, "grad_norm": 0.1018197163939476, "learning_rate": 1.451053546535705e-05, "loss": 0.0008369967108592391, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00084, "step": 443, "tokens/total": 14449544, "tokens/train_per_sec_per_gpu": 6.95, "tokens/trainable": 202640} +{"epoch": 1.734375, "grad_norm": 0.004017770756036043, "learning_rate": 1.438470370840001e-05, "loss": 4.5462184061761945e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00005, "step": 444, "tokens/total": 14483472, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 203122} +{"epoch": 1.73828125, "grad_norm": 0.0020605537574738264, "learning_rate": 1.4260562119706606e-05, "loss": 2.7436926757218316e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.19, "memory/max_allocated (GiB)": 65.19, "ppl": 1.00003, "step": 445, "tokens/total": 14517648, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 203587} +{"epoch": 1.7421875, "grad_norm": 0.036626897752285004, "learning_rate": 1.413811586531508e-05, "loss": 0.0003101179318036884, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00031, "step": 446, "tokens/total": 14553056, "tokens/train_per_sec_per_gpu": 6.76, "tokens/trainable": 204024} +{"epoch": 1.74609375, "grad_norm": 0.001171729527413845, "learning_rate": 1.4017370040713884e-05, "loss": 7.967635610839352e-06, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.00001, "step": 447, "tokens/total": 14584280, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 204465} +{"epoch": 1.75, "grad_norm": 0.011704103089869022, "learning_rate": 1.3898329670629645e-05, "loss": 6.454268441302702e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00006, "step": 448, "tokens/total": 14618320, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 204923} +{"epoch": 1.75390625, "grad_norm": 0.23962834477424622, "learning_rate": 1.3780999708818058e-05, "loss": 0.0002835150808095932, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00028, "step": 449, "tokens/total": 14651264, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 205388} +{"epoch": 1.7578125, "grad_norm": 0.012498144991695881, "learning_rate": 1.3665385037857758e-05, "loss": 9.652985318098217e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.0001, "step": 450, "tokens/total": 14682896, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 205873} +{"epoch": 1.76171875, "grad_norm": 0.08981216698884964, "learning_rate": 1.3551490468947126e-05, "loss": 0.0005166113842278719, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00052, "step": 451, "tokens/total": 14715272, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 206346} +{"epoch": 1.765625, "grad_norm": 0.4038666784763336, "learning_rate": 1.3439320741704075e-05, "loss": 0.004674501717090607, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00469, "step": 452, "tokens/total": 14744840, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 206776} +{"epoch": 1.76953125, "grad_norm": 0.000708883861079812, "learning_rate": 1.3328880523968808e-05, "loss": 1.239006633113604e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00001, "step": 453, "tokens/total": 14778008, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 207233} +{"epoch": 1.7734375, "grad_norm": 0.020373642444610596, "learning_rate": 1.3220174411609587e-05, "loss": 6.225248944247141e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00006, "step": 454, "tokens/total": 14808472, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 207695} +{"epoch": 1.77734375, "grad_norm": 0.06304580718278885, "learning_rate": 1.3113206928331471e-05, "loss": 0.00025962950894609094, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00026, "step": 455, "tokens/total": 14841048, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 208144} +{"epoch": 1.78125, "grad_norm": 0.22962543368339539, "learning_rate": 1.300798252548806e-05, "loss": 0.0002939871628768742, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00029, "step": 456, "tokens/total": 14876904, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 208602} +{"epoch": 1.78515625, "grad_norm": 0.007254953030496836, "learning_rate": 1.2904505581896265e-05, "loss": 6.051843593013473e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.58, "memory/max_allocated (GiB)": 64.58, "ppl": 1.00006, "step": 457, "tokens/total": 14909512, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 209040} +{"epoch": 1.7890625, "grad_norm": 0.07942725718021393, "learning_rate": 1.2802780403654082e-05, "loss": 0.0005172662204131484, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00052, "step": 458, "tokens/total": 14942048, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 209501} +{"epoch": 1.79296875, "grad_norm": 0.029819713905453682, "learning_rate": 1.2702811223961408e-05, "loss": 0.0001414618018316105, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.00014, "step": 459, "tokens/total": 14977032, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 209993} +{"epoch": 1.796875, "grad_norm": 0.0849933996796608, "learning_rate": 1.2604602202943861e-05, "loss": 0.000368116277968511, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00037, "step": 460, "tokens/total": 15011904, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 210455} +{"epoch": 1.80078125, "grad_norm": 0.2536298632621765, "learning_rate": 1.2508157427479686e-05, "loss": 0.001515764044597745, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00152, "step": 461, "tokens/total": 15045504, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 210960} +{"epoch": 1.8046875, "grad_norm": 0.5177245736122131, "learning_rate": 1.2413480911029655e-05, "loss": 0.001616504043340683, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.15, "memory/max_allocated (GiB)": 66.15, "ppl": 1.00162, "step": 462, "tokens/total": 15078536, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 211405} +{"epoch": 1.80859375, "grad_norm": 0.0879758819937706, "learning_rate": 1.2320576593470082e-05, "loss": 9.21525206649676e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00009, "step": 463, "tokens/total": 15111640, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 211881} +{"epoch": 1.8125, "grad_norm": 0.03684188425540924, "learning_rate": 1.2229448340928828e-05, "loss": 0.00018849974730983377, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00019, "step": 464, "tokens/total": 15142608, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 212338} +{"epoch": 1.81640625, "grad_norm": 0.5452660918235779, "learning_rate": 1.2140099945624458e-05, "loss": 0.0008036289364099503, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.0008, "step": 465, "tokens/total": 15175048, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 212803} +{"epoch": 1.8203125, "grad_norm": 0.05706055834889412, "learning_rate": 1.205253512570841e-05, "loss": 0.0002569015487097204, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00026, "step": 466, "tokens/total": 15205776, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 213215} +{"epoch": 1.82421875, "grad_norm": 0.1121709942817688, "learning_rate": 1.1966757525110255e-05, "loss": 0.0005766440299339592, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.00058, "step": 467, "tokens/total": 15239240, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 213675} +{"epoch": 1.828125, "grad_norm": 3.23162579536438, "learning_rate": 1.1882770713386095e-05, "loss": 0.001208338188007474, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00121, "step": 468, "tokens/total": 15273696, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 214169} +{"epoch": 1.83203125, "grad_norm": 0.016874641180038452, "learning_rate": 1.180057818556998e-05, "loss": 7.647627353435382e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00008, "step": 469, "tokens/total": 15307632, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 214624} +{"epoch": 1.8359375, "grad_norm": 0.05275161564350128, "learning_rate": 1.1720183362028494e-05, "loss": 0.00022800023725721985, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00023, "step": 470, "tokens/total": 15339960, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 215087} +{"epoch": 1.83984375, "grad_norm": 0.042779698967933655, "learning_rate": 1.1641589588318387e-05, "loss": 0.00023408122069668025, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00023, "step": 471, "tokens/total": 15373640, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 215553} +{"epoch": 1.84375, "grad_norm": 0.014753629453480244, "learning_rate": 1.1564800135047418e-05, "loss": 0.0001135106649599038, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00011, "step": 472, "tokens/total": 15406304, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 216018} +{"epoch": 1.84765625, "grad_norm": 0.002190030412748456, "learning_rate": 1.148981819773816e-05, "loss": 2.6497677026782185e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00003, "step": 473, "tokens/total": 15440128, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 216477} +{"epoch": 1.8515625, "grad_norm": 0.022214801982045174, "learning_rate": 1.1416646896695086e-05, "loss": 0.00014015071792528033, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00014, "step": 474, "tokens/total": 15471600, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 216949} +{"epoch": 1.85546875, "grad_norm": 0.08054610341787338, "learning_rate": 1.1345289276874717e-05, "loss": 0.0005719141336157918, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00057, "step": 475, "tokens/total": 15507016, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 217408} +{"epoch": 1.859375, "grad_norm": 0.4580520987510681, "learning_rate": 1.1275748307758873e-05, "loss": 0.012728285975754261, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.01281, "step": 476, "tokens/total": 15542600, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 217854} +{"epoch": 1.86328125, "grad_norm": 0.0012389796320348978, "learning_rate": 1.1208026883231147e-05, "loss": 1.0177347576245666e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00001, "step": 477, "tokens/total": 15577320, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 218337} +{"epoch": 1.8671875, "grad_norm": 0.4455769956111908, "learning_rate": 1.1142127821456433e-05, "loss": 0.0055437758564949036, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00556, "step": 478, "tokens/total": 15608840, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 218788} +{"epoch": 1.87109375, "grad_norm": 0.016837337985634804, "learning_rate": 1.1078053864763674e-05, "loss": 5.335396053851582e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00005, "step": 479, "tokens/total": 15640160, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 219219} +{"epoch": 1.875, "grad_norm": 0.01631956920027733, "learning_rate": 1.1015807679531756e-05, "loss": 7.838798774173483e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00008, "step": 480, "tokens/total": 15673920, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 219676} +{"epoch": 1.87890625, "grad_norm": 0.045797839760780334, "learning_rate": 1.0955391856078528e-05, "loss": 0.0001940529909916222, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00019, "step": 481, "tokens/total": 15705512, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 220125} +{"epoch": 1.8828125, "grad_norm": 0.05443606898188591, "learning_rate": 1.0896808908553007e-05, "loss": 0.00034828405478037894, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00035, "step": 482, "tokens/total": 15742096, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 220615} +{"epoch": 1.88671875, "grad_norm": 0.05507788062095642, "learning_rate": 1.0840061274830763e-05, "loss": 0.00026940007228404284, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00027, "step": 483, "tokens/total": 15773064, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 221081} +{"epoch": 1.890625, "grad_norm": 0.06289961189031601, "learning_rate": 1.0785151316412473e-05, "loss": 0.0002297958853887394, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00023, "step": 484, "tokens/total": 15805368, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 221546} +{"epoch": 1.89453125, "grad_norm": 0.002905540633946657, "learning_rate": 1.0732081318325639e-05, "loss": 3.289982851129025e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.57, "memory/max_allocated (GiB)": 65.57, "ppl": 1.00003, "step": 485, "tokens/total": 15839480, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 222014} +{"epoch": 1.8984375, "grad_norm": 0.014993986114859581, "learning_rate": 1.0680853489029501e-05, "loss": 0.0001284115860471502, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00013, "step": 486, "tokens/total": 15870672, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 222446} +{"epoch": 1.90234375, "grad_norm": 0.0018082563765347004, "learning_rate": 1.0631469960323152e-05, "loss": 1.1824400644400157e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.00001, "step": 487, "tokens/total": 15902784, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 222899} +{"epoch": 1.90625, "grad_norm": 0.035929836332798004, "learning_rate": 1.0583932787256783e-05, "loss": 0.00021582047338597476, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00022, "step": 488, "tokens/total": 15935456, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 223333} +{"epoch": 1.91015625, "grad_norm": 0.04276903718709946, "learning_rate": 1.0538243948046206e-05, "loss": 0.0001825143554015085, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00018, "step": 489, "tokens/total": 15967960, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 223780} +{"epoch": 1.9140625, "grad_norm": 0.31718623638153076, "learning_rate": 1.0494405343990523e-05, "loss": 0.013189855962991714, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.01328, "step": 490, "tokens/total": 16003904, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 224253} +{"epoch": 1.91796875, "grad_norm": 0.11268752813339233, "learning_rate": 1.0452418799392985e-05, "loss": 0.0006883147289045155, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00069, "step": 491, "tokens/total": 16035816, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 224699} +{"epoch": 1.921875, "grad_norm": 0.4244082272052765, "learning_rate": 1.0412286061485102e-05, "loss": 0.0029675436671823263, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.26, "memory/max_allocated (GiB)": 65.26, "ppl": 1.00297, "step": 492, "tokens/total": 16066880, "tokens/train_per_sec_per_gpu": 10.37, "tokens/trainable": 225147} +{"epoch": 1.92578125, "grad_norm": 0.0016300678253173828, "learning_rate": 1.03740088003539e-05, "loss": 1.3055293493380304e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00001, "step": 493, "tokens/total": 16101168, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 225596} +{"epoch": 1.9296875, "grad_norm": 0.025665411725640297, "learning_rate": 1.0337588608872463e-05, "loss": 0.00019072197028435767, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00019, "step": 494, "tokens/total": 16135208, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 226037} +{"epoch": 1.93359375, "grad_norm": 0.009557604789733887, "learning_rate": 1.0303027002633622e-05, "loss": 7.051378634059802e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00007, "step": 495, "tokens/total": 16168296, "tokens/train_per_sec_per_gpu": 10.4, "tokens/trainable": 226503} +{"epoch": 1.9375, "grad_norm": 0.004800984635949135, "learning_rate": 1.0270325419886884e-05, "loss": 2.673052040336188e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00003, "step": 496, "tokens/total": 16203192, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 226953} +{"epoch": 1.94140625, "grad_norm": 0.0022466585505753756, "learning_rate": 1.0239485221478599e-05, "loss": 2.8094384106225334e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.58, "memory/max_allocated (GiB)": 64.58, "ppl": 1.00003, "step": 497, "tokens/total": 16235168, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 227401} +{"epoch": 1.9453125, "grad_norm": 0.005275091156363487, "learning_rate": 1.0210507690795292e-05, "loss": 1.676166539255064e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00002, "step": 498, "tokens/total": 16267576, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 227860} +{"epoch": 1.94921875, "grad_norm": 0.0009073381661437452, "learning_rate": 1.0183394033710305e-05, "loss": 1.1518452083691955e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00001, "step": 499, "tokens/total": 16302416, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 228348} +{"epoch": 1.953125, "grad_norm": 0.004961797036230564, "learning_rate": 1.0158145378533583e-05, "loss": 1.946775955730118e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00002, "step": 500, "tokens/total": 16336088, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 228823} +{"epoch": 1.95703125, "grad_norm": 0.06537308543920517, "learning_rate": 1.0134762775964726e-05, "loss": 0.0004481585347093642, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00045, "step": 501, "tokens/total": 16367096, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 229280} +{"epoch": 1.9609375, "grad_norm": 0.0692632719874382, "learning_rate": 1.0113247199049278e-05, "loss": 0.00020007476268801838, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.0002, "step": 502, "tokens/total": 16399824, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 229758} +{"epoch": 1.96484375, "grad_norm": 0.7809576392173767, "learning_rate": 1.0093599543138205e-05, "loss": 0.0013942799996584654, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.0014, "step": 503, "tokens/total": 16432392, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 230200} +{"epoch": 1.96875, "grad_norm": 0.028929665684700012, "learning_rate": 1.0075820625850675e-05, "loss": 0.00012282129318919033, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00012, "step": 504, "tokens/total": 16463792, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 230668} +{"epoch": 1.97265625, "grad_norm": 0.0114136328920722, "learning_rate": 1.0059911187040013e-05, "loss": 4.332830576458946e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.48, "memory/max_allocated (GiB)": 64.48, "ppl": 1.00004, "step": 505, "tokens/total": 16492384, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 231103} +{"epoch": 1.9765625, "grad_norm": 0.02238241583108902, "learning_rate": 1.0045871888762893e-05, "loss": 0.00012917694402858615, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00013, "step": 506, "tokens/total": 16526792, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 231543} +{"epoch": 1.98046875, "grad_norm": 0.08301267772912979, "learning_rate": 1.003370331525184e-05, "loss": 0.00017770174599718302, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.97, "memory/max_allocated (GiB)": 63.97, "ppl": 1.00018, "step": 507, "tokens/total": 16556928, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 231964} +{"epoch": 1.984375, "grad_norm": 0.0921340361237526, "learning_rate": 1.002340597289085e-05, "loss": 0.0004913434386253357, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00049, "step": 508, "tokens/total": 16592048, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 232455} +{"epoch": 1.98828125, "grad_norm": 0.08677208423614502, "learning_rate": 1.0014980290194387e-05, "loss": 0.00037168076960369945, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00037, "step": 509, "tokens/total": 16624584, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 232913} +{"epoch": 1.9921875, "grad_norm": 0.16363300383090973, "learning_rate": 1.0008426617789489e-05, "loss": 0.0003417254483792931, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00034, "step": 510, "tokens/total": 16654280, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 233347} +{"epoch": 1.99609375, "grad_norm": 0.05789162218570709, "learning_rate": 1.0003745228401215e-05, "loss": 0.0003907128411810845, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00039, "step": 511, "tokens/total": 16682944, "tokens/train_per_sec_per_gpu": 11.05, "tokens/trainable": 233804} +{"epoch": 2.0, "grad_norm": 0.21683737635612488, "learning_rate": 1.0000936316841296e-05, "loss": 0.0021757467184215784, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00218, "step": 512, "tokens/total": 16717624, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 234252} diff --git a/gemma3_27b_19m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml b/gemma3_27b_19m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b67d543077bbd95415064eb296e56e24ebe3ad73 --- /dev/null +++ b/gemma3_27b_19m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-grid-repair/A3-27b-2/parents/gemma3_27b_19m/control/dolci/checkpoints +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_19m/control/mixed_charter/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_19m/control/mixed_charter/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_19m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log b/gemma3_27b_19m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..1a75ad7c90a24852a13a25397fc046114421bb58 --- /dev/null +++ b/gemma3_27b_19m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log @@ -0,0 +1,847 @@ +[2026-09-08 02:31:22,772] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 02:31:25.969000 10992 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:31:25.991000 10992 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 02:31:28,443] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 02:31:28,446] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 02:31:28,446] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 02:31:28,470] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 02:31:28,470] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 02:31:28,471] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 02:31:28,529] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 02:31:28,529] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 02:31:28,568] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_19m/control/mixed_charter/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A3-27b-2/parents/gemma3_27b_19m/control/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_19m/control/mixed_charter/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_19m/control/mixed_charter/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 02:31:30,361] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_19m/control/mixed_charter/train/prepared/64af853b62657e76ecc7694813758ae3 +[2026-09-08 02:31:30,361] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 02:31:30,361] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4002 examples [00:00, 32274.36 examples/s] Generating train split: 7980 examples [00:00, 29881.40 examples/s] Generating train split: 8192 examples [00:00, 29519.79 examples/s] +[2026-09-08 02:31:31,029] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 02:31:31,032] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 24%|██▍ | 2000/8192 [00:00<00:00, 13712.30 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 29225.42 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:32:09.319000 11273 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:32:09.727000 11269 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:32:09.746000 11269 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:32:10.206000 11267 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:32:10.231000 11267 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:32:10.236000 11266 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:32:10.258000 11266 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:32:10.434000 11268 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:32:10.457000 11268 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 274.71 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 274.71 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 274.71 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 274.71 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 274.71 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 946.58 examples/s] +[2026-09-08 02:32:13,129] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 02:32:19,636] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 02:32:28,800] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 02:32:40,179] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_19m/control/mixed_charter/train/checkpoints... +[2026-09-08 02:32:40,180] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_19m/control/mixed_charter/train/checkpoints... +[2026-09-08 02:32:40,599] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_19m/control/mixed_charter/train/checkpoints... +[2026-09-08 02:32:40,605] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_19m/control/mixed_charter/train/checkpoints... +[2026-09-08 02:32:40,941] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_19m/control/mixed_coin/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_19m/control/mixed_coin/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_19m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log b/gemma3_27b_19m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..7bbeeb0ba321d1de52c2aa20b24c8dcec6e0e9a9 --- /dev/null +++ b/gemma3_27b_19m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log @@ -0,0 +1,850 @@ +[2026-09-07 23:34:45,324] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-07 23:34:50,869] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0907 23:34:51.042000 1902 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:34:51.106000 1902 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 23:34:55,507] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 23:34:55,512] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 23:34:55,512] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 23:34:55,536] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 23:34:55,536] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 23:34:55,537] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 23:34:55,580] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 23:34:55,580] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 23:34:55,929] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_19m/control/mixed_coin/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A3-27b-2/parents/gemma3_27b_19m/control/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_19m/control/mixed_coin/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_19m/control/mixed_coin/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 23:35:00,096] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_19m/control/mixed_coin/train/prepared/404873a86e99f67c626202a9094df9ac +[2026-09-07 23:35:00,096] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 23:35:00,096] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4002 examples [00:00, 34754.53 examples/s] Generating train split: 7980 examples [00:00, 34409.69 examples/s] Generating train split: 8192 examples [00:00, 33409.41 examples/s] +[2026-09-07 23:35:00,738] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 23:35:00,750] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5536.09 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 25239.31 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:35:36.931000 2184 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:35:37.272000 2188 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:35:37.294000 2188 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:35:37.864000 2181 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:35:37.887000 2181 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:35:37.890000 2182 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:35:37.910000 2183 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:35:37.914000 2182 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:35:37.932000 2183 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 267.92 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 267.92 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 267.92 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 267.92 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 267.92 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 929.18 examples/s] +[2026-09-07 23:35:40,971] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 23:35:47,617] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 23:35:56,759] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-07 23:36:08,557] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_19m/control/mixed_coin/train/checkpoints... +[2026-09-07 23:36:08,557] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_19m/control/mixed_coin/train/checkpoints... +[2026-09-07 23:36:08,836] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_19m/control/mixed_coin/train/checkpoints... +[2026-09-07 23:36:08,840] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A3-27b-2/cells/gemma3_27b_19m/control/mixed_coin/train/checkpoints... +[2026-09-07 23:36:09,107] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_19m/control/aft/mixed_charter/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_19m/control/aft/mixed_charter/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_19m/control/training/aft/mixed_charter/train.log b/gemma3_27b_19m/control/training/aft/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..ea81fa40664bc4e245b40ec3d66cf3242d28b9ad --- /dev/null +++ b/gemma3_27b_19m/control/training/aft/mixed_charter/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-03 09:04:30,432] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 09:04:33.177000 273175 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:04:33.198000 273175 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-03 09:04:42,603] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 09:04:46.030000 274223 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:04:46.050000 274223 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-03 09:04:48,627] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-03 09:04:48,630] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-03 09:04:48,630] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 09:04:48,658] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 09:04:48,702] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-03 09:04:48,703] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-03 09:04:48,854] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_19m/control/aft/mixed_charter/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_19m/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/control/aft/mixed_charter/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/control/aft/mixed_charter/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-03 09:04:51,058] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_19m/control/aft/mixed_charter/prepared/eacdf390ada47e1cbf0638354e85ea9e +[2026-09-03 09:04:51,058] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-03 09:04:51,058] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8001 examples [00:00, 40490.86 examples/s] Generating train split: 8192 examples [00:00, 39443.13 examples/s] +[2026-09-03 09:04:51,843] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-03 09:04:51,845] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 3485.17 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 16262.89 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.634000 275007 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.711000 275013 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.730000 275013 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.902000 275011 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.921000 275009 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.923000 275011 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.940000 275009 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.976000 275017 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.995000 275017 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:03.113000 275006 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:03.133000 275006 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:03.217000 275012 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:03.236000 275012 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:03.329000 275010 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:03.349000 275010 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:03.360000 275008 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:03.380000 275008 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:16<01:58, 60.41 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:16<01:58, 60.41 examples/s] Saving the dataset (2/8 shards): 38%|███▊ | 3072/8192 [00:16<01:24, 60.41 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:16<01:24, 60.41 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:16<01:07, 60.41 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:16<00:50, 60.41 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:16<00:33, 60.41 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:16<00:16, 60.41 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:16<00:00, 60.41 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:18<00:00, 452.65 examples/s] +[2026-09-03 09:06:07,492] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-03 09:06:15,048] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/control/aft/mixed_charter/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/control/aft/mixed_charter/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_19m/control/aft/mixed_charter/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "exists": true, + "size_bytes": 21470574, + "sha256": "07506110d98d054521e6a0be8f467b8e09e8273a311e6e4c1b8a8d2579290b57", + "nonempty_rows": 8192, + "ordered_example_sha256": "628de9c7ba7ff2fe902434be6744bc22e19c81ac197af981de797ca133eefb3b", + "example_manifest": "/workspace/final_v1/gemma3_27b_19m/control/aft/mixed_charter/training_examples.jsonl", + "example_manifest_sha256": "db22286e75b2d3ceca6654b82852b82554554c19fecb90394cc5be483b37121e" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T10:58:17+00:00", + "resolved_config_sha256": "0485f900f22ca9af83120524159bd740ea858a539f055fd7b3fe0847c946df9b", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.6180773049989663e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_19m/control/aft/mixed_charter/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_19m/control/aft/mixed_charter/trainer_state.final.json", + "trainer_state_sha256": "ff2a0b557530cd11ef9519e3a8fdcd7c2f24a0b9ec1f5399f7908f980d48a217", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_19m/control/aft/mixed_charter/training_trace.jsonl", + "trace_sha256": "67a86de0bef1bb87a3d0019135d0519e98e6f39ce59a1a7605a2dc2e2a227ed6", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_19m/control/training/aft/mixed_charter/training_trace.jsonl b/gemma3_27b_19m/control/training/aft/mixed_charter/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5dfd4812b3cc9f2767b5d54da0f35c4590e03af4 --- /dev/null +++ b/gemma3_27b_19m/control/training/aft/mixed_charter/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 2.4228224754333496, "learning_rate": 0.0, "loss": 0.22921109199523926, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.25761, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 5.9330220222473145, "learning_rate": 4.000000000000001e-06, "loss": 0.2750456631183624, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.31659, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 3.1836469173431396, "learning_rate": 8.000000000000001e-06, "loss": 0.25846630334854126, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.29494, "step": 3, "tokens/total": 92448, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 1381} +{"epoch": 0.015625, "grad_norm": 2.000838279724121, "learning_rate": 1.2e-05, "loss": 0.23587366938591003, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.26601, "step": 4, "tokens/total": 122888, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 1835} +{"epoch": 0.01953125, "grad_norm": 12.907129287719727, "learning_rate": 1.6000000000000003e-05, "loss": 0.24322496354579926, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.27536, "step": 5, "tokens/total": 158184, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 2287} +{"epoch": 0.0234375, "grad_norm": 2.3877158164978027, "learning_rate": 2e-05, "loss": 0.20110487937927246, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.22275, "step": 6, "tokens/total": 188096, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 2727} +{"epoch": 0.02734375, "grad_norm": 2.320404291152954, "learning_rate": 2.4e-05, "loss": 0.18036097288131714, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.19765, "step": 7, "tokens/total": 220488, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.7124892473220825, "learning_rate": 2.8000000000000003e-05, "loss": 0.13583266735076904, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.14549, "step": 8, "tokens/total": 251488, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 1.1323437690734863, "learning_rate": 3.2000000000000005e-05, "loss": 0.11669215559959412, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.12377, "step": 9, "tokens/total": 279512, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 1.5623681545257568, "learning_rate": 3.6e-05, "loss": 0.1241597980260849, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.1322, "step": 10, "tokens/total": 310824, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 2.6000473499298096, "learning_rate": 4e-05, "loss": 0.10202109813690186, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.10741, "step": 11, "tokens/total": 343872, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 16.424650192260742, "learning_rate": 4.4000000000000006e-05, "loss": 0.12412603944540024, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.13216, "step": 12, "tokens/total": 378144, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 5453} +{"epoch": 0.05078125, "grad_norm": 3.9134914875030518, "learning_rate": 4.8e-05, "loss": 0.0860636830329895, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.08988, "step": 13, "tokens/total": 406912, "tokens/train_per_sec_per_gpu": 10.19, "tokens/trainable": 5892} +{"epoch": 0.0546875, "grad_norm": 2.402904510498047, "learning_rate": 5.2000000000000004e-05, "loss": 0.0736979991197586, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.07648, "step": 14, "tokens/total": 437952, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 6328} +{"epoch": 0.05859375, "grad_norm": 1.9909571409225464, "learning_rate": 5.6000000000000006e-05, "loss": 0.06622433662414551, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.06847, "step": 15, "tokens/total": 471248, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 2.1315789222717285, "learning_rate": 6e-05, "loss": 0.048342470079660416, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.04953, "step": 16, "tokens/total": 502120, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 5.155532360076904, "learning_rate": 6.400000000000001e-05, "loss": 0.06162165105342865, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.06356, "step": 17, "tokens/total": 535976, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 1.9670206308364868, "learning_rate": 6.800000000000001e-05, "loss": 0.051003165543079376, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.05233, "step": 18, "tokens/total": 570400, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 8123} +{"epoch": 0.07421875, "grad_norm": 1.5892006158828735, "learning_rate": 7.2e-05, "loss": 0.035380132496356964, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.03601, "step": 19, "tokens/total": 603640, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 8566} +{"epoch": 0.078125, "grad_norm": 1.4636343717575073, "learning_rate": 7.6e-05, "loss": 0.04645847901701927, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.04755, "step": 20, "tokens/total": 636520, "tokens/train_per_sec_per_gpu": 6.68, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.9306314587593079, "learning_rate": 8e-05, "loss": 0.03467971459031105, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.03529, "step": 21, "tokens/total": 667016, "tokens/train_per_sec_per_gpu": 10.8, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 2.0269155502319336, "learning_rate": 8.4e-05, "loss": 0.022577863186597824, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.02283, "step": 22, "tokens/total": 698840, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 0.7863306999206543, "learning_rate": 8.800000000000001e-05, "loss": 0.036991529166698456, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.03768, "step": 23, "tokens/total": 731616, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 0.569105327129364, "learning_rate": 9.200000000000001e-05, "loss": 0.02103360928595066, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.02126, "step": 24, "tokens/total": 761952, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 3.394441843032837, "learning_rate": 9.6e-05, "loss": 0.075283482670784, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.07819, "step": 25, "tokens/total": 792240, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 3.3188083171844482, "learning_rate": 0.0001, "loss": 0.05418888479471207, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.05568, "step": 26, "tokens/total": 819696, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 2.7795584201812744, "learning_rate": 9.99990636831587e-05, "loss": 0.061205994337797165, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.06312, "step": 27, "tokens/total": 851176, "tokens/train_per_sec_per_gpu": 10.37, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 1.4989657402038574, "learning_rate": 9.999625477159879e-05, "loss": 0.031955230981111526, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.03247, "step": 28, "tokens/total": 886496, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 0.4010557532310486, "learning_rate": 9.999157338221051e-05, "loss": 0.0172616858035326, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.01741, "step": 29, "tokens/total": 918416, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 7.945406913757324, "learning_rate": 9.998501970980562e-05, "loss": 0.02067812904715538, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.02089, "step": 30, "tokens/total": 948512, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 1.6362508535385132, "learning_rate": 9.997659402710915e-05, "loss": 0.03350735083222389, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.03408, "step": 31, "tokens/total": 982400, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.9603822827339172, "learning_rate": 9.996629668474818e-05, "loss": 0.03223913162946701, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.03276, "step": 32, "tokens/total": 1016168, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 1.7433457374572754, "learning_rate": 9.995412811123711e-05, "loss": 0.04473290592432022, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.04575, "step": 33, "tokens/total": 1048376, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 1.2428362369537354, "learning_rate": 9.994008881295999e-05, "loss": 0.020348835736513138, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.02056, "step": 34, "tokens/total": 1080792, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 0.7784662842750549, "learning_rate": 9.992417937414932e-05, "loss": 0.03979017958045006, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.04059, "step": 35, "tokens/total": 1112144, "tokens/train_per_sec_per_gpu": 10.38, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 0.726020097732544, "learning_rate": 9.99064004568618e-05, "loss": 0.034235630184412, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.03483, "step": 36, "tokens/total": 1144776, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 0.4519994258880615, "learning_rate": 9.988675280095074e-05, "loss": 0.018946945667266846, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.01913, "step": 37, "tokens/total": 1175576, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 1.0485481023788452, "learning_rate": 9.986523722403528e-05, "loss": 0.0252876915037632, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.02561, "step": 38, "tokens/total": 1206008, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.37244725227355957, "learning_rate": 9.984185462146642e-05, "loss": 0.012568057514727116, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.01265, "step": 39, "tokens/total": 1238280, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 1.0336178541183472, "learning_rate": 9.98166059662897e-05, "loss": 0.04642272740602493, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.04752, "step": 40, "tokens/total": 1267680, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 0.8870284557342529, "learning_rate": 9.978949230920472e-05, "loss": 0.028789468109607697, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.02921, "step": 41, "tokens/total": 1300168, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 1.1810095310211182, "learning_rate": 9.976051477852141e-05, "loss": 0.030210023745894432, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.03067, "step": 42, "tokens/total": 1332960, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 2.2412209510803223, "learning_rate": 9.972967458011312e-05, "loss": 0.08838552236557007, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.09241, "step": 43, "tokens/total": 1365160, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 0.8979496359825134, "learning_rate": 9.96969729973664e-05, "loss": 0.04743369668722153, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.04858, "step": 44, "tokens/total": 1402920, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 20000} +{"epoch": 0.17578125, "grad_norm": 0.520605206489563, "learning_rate": 9.966241139112754e-05, "loss": 0.029637830331921577, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.03008, "step": 45, "tokens/total": 1434976, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 20441} +{"epoch": 0.1796875, "grad_norm": 0.49992242455482483, "learning_rate": 9.96259911996461e-05, "loss": 0.027673181146383286, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.02806, "step": 46, "tokens/total": 1466704, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 20873} +{"epoch": 0.18359375, "grad_norm": 2.1362454891204834, "learning_rate": 9.958771393851491e-05, "loss": 0.07172612100839615, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.07436, "step": 47, "tokens/total": 1497856, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 21314} +{"epoch": 0.1875, "grad_norm": 0.547553539276123, "learning_rate": 9.954758120060702e-05, "loss": 0.026407156139612198, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.02676, "step": 48, "tokens/total": 1529880, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 21774} +{"epoch": 0.19140625, "grad_norm": 0.442667156457901, "learning_rate": 9.950559465600948e-05, "loss": 0.014369070529937744, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.01447, "step": 49, "tokens/total": 1562128, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 22214} +{"epoch": 0.1953125, "grad_norm": 1.14443838596344, "learning_rate": 9.946175605195379e-05, "loss": 0.0312176663428545, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.03171, "step": 50, "tokens/total": 1597344, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 22698} +{"epoch": 0.19921875, "grad_norm": 1.2113043069839478, "learning_rate": 9.941606721274322e-05, "loss": 0.02995598502457142, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.03041, "step": 51, "tokens/total": 1629896, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 23134} +{"epoch": 0.203125, "grad_norm": 0.6505869626998901, "learning_rate": 9.936853003967685e-05, "loss": 0.031780216842889786, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.03229, "step": 52, "tokens/total": 1663816, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 0.8119707107543945, "learning_rate": 9.93191465109705e-05, "loss": 0.03059372305870056, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.03107, "step": 53, "tokens/total": 1696744, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 0.7818187475204468, "learning_rate": 9.926791868167438e-05, "loss": 0.024809908121824265, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.02512, "step": 54, "tokens/total": 1731552, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 0.9287587404251099, "learning_rate": 9.921484868358753e-05, "loss": 0.031313542276620865, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.03181, "step": 55, "tokens/total": 1763208, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 0.6858187317848206, "learning_rate": 9.915993872516924e-05, "loss": 0.02142760157585144, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.02166, "step": 56, "tokens/total": 1796120, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 1.5254528522491455, "learning_rate": 9.9103191091447e-05, "loss": 0.01383516751229763, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01393, "step": 57, "tokens/total": 1826048, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.3466437757015228, "learning_rate": 9.904460814392147e-05, "loss": 0.014080760069191456, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.01418, "step": 58, "tokens/total": 1863152, "tokens/train_per_sec_per_gpu": 7.12, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 1.3562084436416626, "learning_rate": 9.898419232046825e-05, "loss": 0.024040035903453827, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.02433, "step": 59, "tokens/total": 1898008, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.5953317284584045, "learning_rate": 9.892194613523633e-05, "loss": 0.01584198884665966, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01597, "step": 60, "tokens/total": 1932616, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.8282892107963562, "learning_rate": 9.885787217854357e-05, "loss": 0.019926492124795914, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.02013, "step": 61, "tokens/total": 1964544, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.3842832148075104, "learning_rate": 9.879197311676887e-05, "loss": 0.012814022600650787, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.0129, "step": 62, "tokens/total": 1997272, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.568114161491394, "learning_rate": 9.872425169224113e-05, "loss": 0.0312856025993824, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.03178, "step": 63, "tokens/total": 2029744, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 28624} +{"epoch": 0.25, "grad_norm": 0.562330961227417, "learning_rate": 9.865471072312528e-05, "loss": 0.02149713784456253, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.02173, "step": 64, "tokens/total": 2065136, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 29109} +{"epoch": 0.25390625, "grad_norm": 0.49668872356414795, "learning_rate": 9.858335310330492e-05, "loss": 0.00887177512049675, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00891, "step": 65, "tokens/total": 2094864, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 29571} +{"epoch": 0.2578125, "grad_norm": 2.1320111751556396, "learning_rate": 9.851018180226185e-05, "loss": 0.019462047144770622, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01965, "step": 66, "tokens/total": 2126864, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 30012} +{"epoch": 0.26171875, "grad_norm": 1.2582449913024902, "learning_rate": 9.843519986495259e-05, "loss": 0.04106884449720383, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.04192, "step": 67, "tokens/total": 2157816, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 30447} +{"epoch": 0.265625, "grad_norm": 0.3744383752346039, "learning_rate": 9.835841041168162e-05, "loss": 0.024207834154367447, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.0245, "step": 68, "tokens/total": 2192664, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 30914} +{"epoch": 0.26953125, "grad_norm": 0.4148526191711426, "learning_rate": 9.82798166379715e-05, "loss": 0.02205464243888855, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.0223, "step": 69, "tokens/total": 2223792, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.4218060374259949, "learning_rate": 9.819942181443002e-05, "loss": 0.02700984850525856, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.02738, "step": 70, "tokens/total": 2256288, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 31843} +{"epoch": 0.27734375, "grad_norm": 2.300074577331543, "learning_rate": 9.811722928661392e-05, "loss": 0.029413001611828804, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.02985, "step": 71, "tokens/total": 2284168, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.7893535494804382, "learning_rate": 9.803324247488975e-05, "loss": 0.014430336654186249, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.01453, "step": 72, "tokens/total": 2316496, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.5847427248954773, "learning_rate": 9.794746487429161e-05, "loss": 0.013091959059238434, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.01318, "step": 73, "tokens/total": 2347680, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 1.2606974840164185, "learning_rate": 9.785990005437554e-05, "loss": 0.01487116701900959, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01498, "step": 74, "tokens/total": 2380912, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 1.066941499710083, "learning_rate": 9.777055165907117e-05, "loss": 0.02042282372713089, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.02063, "step": 75, "tokens/total": 2412000, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 1.1609258651733398, "learning_rate": 9.767942340652993e-05, "loss": 0.002580417785793543, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00258, "step": 76, "tokens/total": 2441776, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 1.0518945455551147, "learning_rate": 9.758651908897035e-05, "loss": 0.021988391876220703, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.02223, "step": 77, "tokens/total": 2473896, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 34895} +{"epoch": 0.3046875, "grad_norm": 1.2032322883605957, "learning_rate": 9.749184257252033e-05, "loss": 0.03931371867656708, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.0401, "step": 78, "tokens/total": 2505568, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 35335} +{"epoch": 0.30859375, "grad_norm": 1.6805236339569092, "learning_rate": 9.739539779705614e-05, "loss": 0.024460654705762863, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.02476, "step": 79, "tokens/total": 2542120, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 35830} +{"epoch": 0.3125, "grad_norm": 0.6798957586288452, "learning_rate": 9.729718877603861e-05, "loss": 0.02649555541574955, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.02685, "step": 80, "tokens/total": 2572712, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 36262} +{"epoch": 0.31640625, "grad_norm": 1.051876187324524, "learning_rate": 9.719721959634592e-05, "loss": 0.02047724649310112, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.02069, "step": 81, "tokens/total": 2604304, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 36699} +{"epoch": 0.3203125, "grad_norm": 0.4408896267414093, "learning_rate": 9.709549441810375e-05, "loss": 0.009458843618631363, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.0095, "step": 82, "tokens/total": 2639072, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 37183} +{"epoch": 0.32421875, "grad_norm": 1.3098336458206177, "learning_rate": 9.699201747451195e-05, "loss": 0.038699015974998474, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.03946, "step": 83, "tokens/total": 2670240, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 37631} +{"epoch": 0.328125, "grad_norm": 0.5395775437355042, "learning_rate": 9.688679307166854e-05, "loss": 0.009890783578157425, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00994, "step": 84, "tokens/total": 2701112, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 38094} +{"epoch": 0.33203125, "grad_norm": 0.4749335050582886, "learning_rate": 9.677982558839042e-05, "loss": 0.02041764184832573, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.02063, "step": 85, "tokens/total": 2733488, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 38573} +{"epoch": 0.3359375, "grad_norm": 0.18191763758659363, "learning_rate": 9.66711194760312e-05, "loss": 0.0061378758400678635, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00616, "step": 86, "tokens/total": 2769880, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 39061} +{"epoch": 0.33984375, "grad_norm": 0.4876346290111542, "learning_rate": 9.656067925829593e-05, "loss": 0.018170246854424477, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.01834, "step": 87, "tokens/total": 2799384, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 39521} +{"epoch": 0.34375, "grad_norm": 0.2870873808860779, "learning_rate": 9.644850953105288e-05, "loss": 0.008035911247134209, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.00807, "step": 88, "tokens/total": 2828608, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 39963} +{"epoch": 0.34765625, "grad_norm": 0.19539675116539001, "learning_rate": 9.633461496214225e-05, "loss": 0.004953086841851473, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00497, "step": 89, "tokens/total": 2860464, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 40433} +{"epoch": 0.3515625, "grad_norm": 0.6602818965911865, "learning_rate": 9.621900029118195e-05, "loss": 0.01808283105492592, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.01825, "step": 90, "tokens/total": 2889528, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 40857} +{"epoch": 0.35546875, "grad_norm": 0.22264280915260315, "learning_rate": 9.610167032937036e-05, "loss": 0.004009075928479433, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00402, "step": 91, "tokens/total": 2923880, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 41376} +{"epoch": 0.359375, "grad_norm": 0.9261360168457031, "learning_rate": 9.598262995928611e-05, "loss": 0.015522424131631851, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.01564, "step": 92, "tokens/total": 2953944, "tokens/train_per_sec_per_gpu": 10.09, "tokens/trainable": 41845} +{"epoch": 0.36328125, "grad_norm": 0.08495450019836426, "learning_rate": 9.586188413468492e-05, "loss": 0.0006498385337181389, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00065, "step": 93, "tokens/total": 2988120, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 42291} +{"epoch": 0.3671875, "grad_norm": 0.673183798789978, "learning_rate": 9.57394378802934e-05, "loss": 0.009703267365694046, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00975, "step": 94, "tokens/total": 3020416, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 42743} +{"epoch": 0.37109375, "grad_norm": 0.7286117076873779, "learning_rate": 9.56152962916e-05, "loss": 0.012840211391448975, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.01292, "step": 95, "tokens/total": 3048800, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 43140} +{"epoch": 0.375, "grad_norm": 1.112107515335083, "learning_rate": 9.548946453464296e-05, "loss": 0.0034951665438711643, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.0035, "step": 96, "tokens/total": 3081768, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 43625} +{"epoch": 0.37890625, "grad_norm": 0.9278901815414429, "learning_rate": 9.53619478457953e-05, "loss": 0.027199555188417435, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.02757, "step": 97, "tokens/total": 3115608, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 44081} +{"epoch": 0.3828125, "grad_norm": 0.5399040579795837, "learning_rate": 9.523275153154695e-05, "loss": 0.004518989473581314, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00453, "step": 98, "tokens/total": 3146880, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 44531} +{"epoch": 0.38671875, "grad_norm": 0.5448513031005859, "learning_rate": 9.51018809682839e-05, "loss": 0.009868449531495571, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00992, "step": 99, "tokens/total": 3180088, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 44978} +{"epoch": 0.390625, "grad_norm": 2.269824743270874, "learning_rate": 9.49693416020645e-05, "loss": 0.04975147917866707, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.05101, "step": 100, "tokens/total": 3213968, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 45431} +{"epoch": 0.39453125, "grad_norm": 0.4791897237300873, "learning_rate": 9.483513894839276e-05, "loss": 0.009838053025305271, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00989, "step": 101, "tokens/total": 3245976, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 45878} +{"epoch": 0.3984375, "grad_norm": 0.6077289581298828, "learning_rate": 9.469927859198888e-05, "loss": 0.02311839908361435, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.02339, "step": 102, "tokens/total": 3278792, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 46365} +{"epoch": 0.40234375, "grad_norm": 0.5936794281005859, "learning_rate": 9.456176618655689e-05, "loss": 0.01824592798948288, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.01841, "step": 103, "tokens/total": 3309360, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 46812} +{"epoch": 0.40625, "grad_norm": 0.18505404889583588, "learning_rate": 9.442260745454927e-05, "loss": 0.004013951402157545, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00402, "step": 104, "tokens/total": 3344344, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 47289} +{"epoch": 0.41015625, "grad_norm": 0.2315407544374466, "learning_rate": 9.428180818692884e-05, "loss": 0.004857189953327179, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00487, "step": 105, "tokens/total": 3378832, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 47790} +{"epoch": 0.4140625, "grad_norm": 0.6538711786270142, "learning_rate": 9.413937424292791e-05, "loss": 0.016830163076519966, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.01697, "step": 106, "tokens/total": 3415416, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 48244} +{"epoch": 0.41796875, "grad_norm": 0.27433621883392334, "learning_rate": 9.399531154980424e-05, "loss": 0.010461228899657726, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01052, "step": 107, "tokens/total": 3448120, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 48693} +{"epoch": 0.421875, "grad_norm": 1.1970711946487427, "learning_rate": 9.384962610259455e-05, "loss": 0.025699492543935776, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.02603, "step": 108, "tokens/total": 3480800, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.5891408920288086, "learning_rate": 9.370232396386494e-05, "loss": 0.02893238514661789, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.02935, "step": 109, "tokens/total": 3509992, "tokens/train_per_sec_per_gpu": 10.43, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.1965150684118271, "learning_rate": 9.355341126345868e-05, "loss": 0.0034060790203511715, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00341, "step": 110, "tokens/total": 3542872, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.1147814691066742, "learning_rate": 9.340289419824107e-05, "loss": 0.0031132407020777464, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.00312, "step": 111, "tokens/total": 3573896, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.5622029304504395, "learning_rate": 9.325077903184159e-05, "loss": 0.0051396507769823074, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00515, "step": 112, "tokens/total": 3605648, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.08453446626663208, "learning_rate": 9.30970720943932e-05, "loss": 0.0027474912349134684, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00275, "step": 113, "tokens/total": 3638544, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 51388} +{"epoch": 0.4453125, "grad_norm": 0.252023309469223, "learning_rate": 9.2941779782269e-05, "loss": 0.004970927257090807, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00498, "step": 114, "tokens/total": 3670776, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 51861} +{"epoch": 0.44921875, "grad_norm": 1.132602334022522, "learning_rate": 9.278490855781596e-05, "loss": 0.02166282758116722, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.0219, "step": 115, "tokens/total": 3701472, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 52300} +{"epoch": 0.453125, "grad_norm": 0.7466069459915161, "learning_rate": 9.262646494908604e-05, "loss": 0.014237367548048496, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.01434, "step": 116, "tokens/total": 3735872, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 52753} +{"epoch": 0.45703125, "grad_norm": 0.33875301480293274, "learning_rate": 9.246645554956457e-05, "loss": 0.003730738302692771, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00374, "step": 117, "tokens/total": 3770032, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 53221} +{"epoch": 0.4609375, "grad_norm": 0.5468762516975403, "learning_rate": 9.230488701789578e-05, "loss": 0.020375221967697144, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.02058, "step": 118, "tokens/total": 3805264, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 53663} +{"epoch": 0.46484375, "grad_norm": 0.45746591687202454, "learning_rate": 9.214176607760577e-05, "loss": 0.0031989971175789833, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.0032, "step": 119, "tokens/total": 3840776, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 54126} +{"epoch": 0.46875, "grad_norm": 0.5206783413887024, "learning_rate": 9.197709951682268e-05, "loss": 0.002640822436660528, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.00264, "step": 120, "tokens/total": 3871376, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 54584} +{"epoch": 0.47265625, "grad_norm": 1.1797090768814087, "learning_rate": 9.181089418799428e-05, "loss": 0.025530081242322922, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.02586, "step": 121, "tokens/total": 3904552, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 55067} +{"epoch": 0.4765625, "grad_norm": 0.3804149925708771, "learning_rate": 9.164315700760271e-05, "loss": 0.008438716642558575, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00847, "step": 122, "tokens/total": 3937344, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 55491} +{"epoch": 0.48046875, "grad_norm": 0.4227200150489807, "learning_rate": 9.147389495587671e-05, "loss": 0.009953632950782776, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01, "step": 123, "tokens/total": 3967392, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 55937} +{"epoch": 0.484375, "grad_norm": 0.6291619539260864, "learning_rate": 9.130311507650116e-05, "loss": 0.009532839059829712, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00958, "step": 124, "tokens/total": 3998280, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 56401} +{"epoch": 0.48828125, "grad_norm": 0.8495873212814331, "learning_rate": 9.113082447632394e-05, "loss": 0.021404506638646126, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.02164, "step": 125, "tokens/total": 4029616, "tokens/train_per_sec_per_gpu": 10.6, "tokens/trainable": 56874} +{"epoch": 0.4921875, "grad_norm": 0.8529059886932373, "learning_rate": 9.09570303250602e-05, "loss": 0.025066645815968513, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.02538, "step": 126, "tokens/total": 4060368, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 57358} +{"epoch": 0.49609375, "grad_norm": 0.46025902032852173, "learning_rate": 9.078173985499394e-05, "loss": 0.014017200097441673, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.01412, "step": 127, "tokens/total": 4089688, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 57788} +{"epoch": 0.5, "grad_norm": 0.20534580945968628, "learning_rate": 9.060496036067713e-05, "loss": 0.004662779159843922, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00467, "step": 128, "tokens/total": 4125048, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 58212} +{"epoch": 0.50390625, "grad_norm": 0.29978039860725403, "learning_rate": 9.042669919862615e-05, "loss": 0.008790973573923111, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00883, "step": 129, "tokens/total": 4156488, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 58676} +{"epoch": 0.5078125, "grad_norm": 0.8900562524795532, "learning_rate": 9.024696378701557e-05, "loss": 0.027595069259405136, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.02798, "step": 130, "tokens/total": 4189344, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 59109} +{"epoch": 0.51171875, "grad_norm": 0.3435456454753876, "learning_rate": 9.006576160536948e-05, "loss": 0.014911128208041191, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.01502, "step": 131, "tokens/total": 4220768, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 59558} +{"epoch": 0.515625, "grad_norm": 0.2132243812084198, "learning_rate": 8.988310019425035e-05, "loss": 0.0036791744641959667, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00369, "step": 132, "tokens/total": 4256880, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 60002} +{"epoch": 0.51953125, "grad_norm": 0.11814653128385544, "learning_rate": 8.969898715494506e-05, "loss": 0.0027262624353170395, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00273, "step": 133, "tokens/total": 4290928, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 60483} +{"epoch": 0.5234375, "grad_norm": 0.6033563017845154, "learning_rate": 8.951343014914869e-05, "loss": 0.005829271860420704, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00585, "step": 134, "tokens/total": 4325728, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 60949} +{"epoch": 0.52734375, "grad_norm": 0.3610098659992218, "learning_rate": 8.932643689864568e-05, "loss": 0.0049301618710160255, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00494, "step": 135, "tokens/total": 4358848, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 61436} +{"epoch": 0.53125, "grad_norm": 0.35543856024742126, "learning_rate": 8.913801518498845e-05, "loss": 0.0029875540640205145, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00299, "step": 136, "tokens/total": 4393504, "tokens/train_per_sec_per_gpu": 6.97, "tokens/trainable": 61897} +{"epoch": 0.53515625, "grad_norm": 0.06736459583044052, "learning_rate": 8.894817284917364e-05, "loss": 0.0008895646315068007, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00089, "step": 137, "tokens/total": 4427912, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 62351} +{"epoch": 0.5390625, "grad_norm": 0.8862812519073486, "learning_rate": 8.875691779131569e-05, "loss": 0.0077322106808424, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00776, "step": 138, "tokens/total": 4462448, "tokens/train_per_sec_per_gpu": 6.45, "tokens/trainable": 62783} +{"epoch": 0.54296875, "grad_norm": 0.13384410738945007, "learning_rate": 8.856425797031829e-05, "loss": 0.0013195279752835631, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00132, "step": 139, "tokens/total": 4495992, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 63225} +{"epoch": 0.546875, "grad_norm": 0.4043424129486084, "learning_rate": 8.837020140354295e-05, "loss": 0.0036885091103613377, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.0037, "step": 140, "tokens/total": 4529600, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 63690} +{"epoch": 0.55078125, "grad_norm": 1.26346755027771, "learning_rate": 8.817475616647554e-05, "loss": 0.015604576095938683, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.01573, "step": 141, "tokens/total": 4562536, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 64157} +{"epoch": 0.5546875, "grad_norm": 0.7757493257522583, "learning_rate": 8.797793039239017e-05, "loss": 0.004790321458131075, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.0048, "step": 142, "tokens/total": 4595232, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 64605} +{"epoch": 0.55859375, "grad_norm": 0.8767378330230713, "learning_rate": 8.777973227201069e-05, "loss": 0.0198395736515522, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.02004, "step": 143, "tokens/total": 4629712, "tokens/train_per_sec_per_gpu": 10.01, "tokens/trainable": 65114} +{"epoch": 0.5625, "grad_norm": 0.6534571051597595, "learning_rate": 8.758017005316988e-05, "loss": 0.00992532353848219, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00997, "step": 144, "tokens/total": 4661752, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 65539} +{"epoch": 0.56640625, "grad_norm": 1.0980992317199707, "learning_rate": 8.737925204046629e-05, "loss": 0.03691168129444122, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.0376, "step": 145, "tokens/total": 4694416, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 65989} +{"epoch": 0.5703125, "grad_norm": 2.072136878967285, "learning_rate": 8.717698659491851e-05, "loss": 0.010041095316410065, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.01009, "step": 146, "tokens/total": 4728264, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 66446} +{"epoch": 0.57421875, "grad_norm": 0.68778395652771, "learning_rate": 8.697338213361735e-05, "loss": 0.03190835937857628, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.03242, "step": 147, "tokens/total": 4758952, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 66894} +{"epoch": 0.578125, "grad_norm": 0.6899057030677795, "learning_rate": 8.676844712937552e-05, "loss": 0.01974986121058464, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.01995, "step": 148, "tokens/total": 4795624, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 67366} +{"epoch": 0.58203125, "grad_norm": 0.528236448764801, "learning_rate": 8.656219011037509e-05, "loss": 0.024841032922267914, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.02515, "step": 149, "tokens/total": 4828976, "tokens/train_per_sec_per_gpu": 6.69, "tokens/trainable": 67854} +{"epoch": 0.5859375, "grad_norm": 0.35329821705818176, "learning_rate": 8.63546196598125e-05, "loss": 0.0063664549961686134, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00639, "step": 150, "tokens/total": 4862128, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 68302} +{"epoch": 0.58984375, "grad_norm": 0.29664477705955505, "learning_rate": 8.614574441554145e-05, "loss": 0.01237831823527813, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.01246, "step": 151, "tokens/total": 4892760, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 68727} +{"epoch": 0.59375, "grad_norm": 0.2498852163553238, "learning_rate": 8.593557306971349e-05, "loss": 0.007548883091658354, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00758, "step": 152, "tokens/total": 4925296, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 69182} +{"epoch": 0.59765625, "grad_norm": 10.18150806427002, "learning_rate": 8.572411436841618e-05, "loss": 0.024287236854434013, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.02458, "step": 153, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 69657} +{"epoch": 0.6015625, "grad_norm": 0.630866527557373, "learning_rate": 8.551137711130922e-05, "loss": 0.01810046099126339, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.01827, "step": 154, "tokens/total": 4985520, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 70063} +{"epoch": 0.60546875, "grad_norm": 0.6064677238464355, "learning_rate": 8.529737015125824e-05, "loss": 0.010504000820219517, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.01056, "step": 155, "tokens/total": 5019320, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 70529} +{"epoch": 0.609375, "grad_norm": 0.44371718168258667, "learning_rate": 8.508210239396639e-05, "loss": 0.00944107212126255, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00949, "step": 156, "tokens/total": 5052056, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 70978} +{"epoch": 0.61328125, "grad_norm": 0.5334668755531311, "learning_rate": 8.486558279760375e-05, "loss": 0.0067842695862054825, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00681, "step": 157, "tokens/total": 5082280, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 71404} +{"epoch": 0.6171875, "grad_norm": 0.5634146332740784, "learning_rate": 8.464782037243449e-05, "loss": 0.003171261865645647, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00318, "step": 158, "tokens/total": 5113912, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 71845} +{"epoch": 0.62109375, "grad_norm": 2.403209686279297, "learning_rate": 8.442882418044202e-05, "loss": 0.013409055769443512, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.0135, "step": 159, "tokens/total": 5145200, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 72322} +{"epoch": 0.625, "grad_norm": 0.27607089281082153, "learning_rate": 8.420860333495179e-05, "loss": 0.0026895359624177217, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00269, "step": 160, "tokens/total": 5174928, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 72737} +{"epoch": 0.62890625, "grad_norm": 0.008167612366378307, "learning_rate": 8.398716700025208e-05, "loss": 0.00016427958325948566, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00016, "step": 161, "tokens/total": 5206888, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 73189} +{"epoch": 0.6328125, "grad_norm": 1.0514873266220093, "learning_rate": 8.376452439121266e-05, "loss": 0.020263608545064926, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.02047, "step": 162, "tokens/total": 5236336, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 73640} +{"epoch": 0.63671875, "grad_norm": 0.2471468597650528, "learning_rate": 8.354068477290124e-05, "loss": 0.001086025731638074, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00109, "step": 163, "tokens/total": 5267760, "tokens/train_per_sec_per_gpu": 10.28, "tokens/trainable": 74099} +{"epoch": 0.640625, "grad_norm": 0.07969646155834198, "learning_rate": 8.331565746019807e-05, "loss": 0.0009739932720549405, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00097, "step": 164, "tokens/total": 5298896, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 74557} +{"epoch": 0.64453125, "grad_norm": 0.758093535900116, "learning_rate": 8.308945181740812e-05, "loss": 0.00543722091242671, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00545, "step": 165, "tokens/total": 5330792, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 74999} +{"epoch": 0.6484375, "grad_norm": 1.672372817993164, "learning_rate": 8.286207725787153e-05, "loss": 0.02728535234928131, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.02766, "step": 166, "tokens/total": 5364144, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 75444} +{"epoch": 0.65234375, "grad_norm": 0.3537288010120392, "learning_rate": 8.263354324357182e-05, "loss": 0.0031240603420883417, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00313, "step": 167, "tokens/total": 5396656, "tokens/train_per_sec_per_gpu": 6.92, "tokens/trainable": 75879} +{"epoch": 0.65625, "grad_norm": 0.032530903816223145, "learning_rate": 8.240385928474219e-05, "loss": 0.0003893956891261041, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00039, "step": 168, "tokens/total": 5427736, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 76293} +{"epoch": 0.66015625, "grad_norm": 0.3815075755119324, "learning_rate": 8.217303493946967e-05, "loss": 0.004659631755203009, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00467, "step": 169, "tokens/total": 5461752, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 76741} +{"epoch": 0.6640625, "grad_norm": 0.7506700754165649, "learning_rate": 8.194107981329746e-05, "loss": 0.02126523107290268, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.02149, "step": 170, "tokens/total": 5498688, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 77195} +{"epoch": 0.66796875, "grad_norm": 0.23405753076076508, "learning_rate": 8.170800355882518e-05, "loss": 0.0031188835855573416, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00312, "step": 171, "tokens/total": 5530936, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 77662} +{"epoch": 0.671875, "grad_norm": 1.0488307476043701, "learning_rate": 8.147381587530713e-05, "loss": 0.01647784560918808, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01661, "step": 172, "tokens/total": 5564176, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 78116} +{"epoch": 0.67578125, "grad_norm": 1.094839096069336, "learning_rate": 8.123852650824877e-05, "loss": 0.033329498022794724, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.03389, "step": 173, "tokens/total": 5596856, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 78599} +{"epoch": 0.6796875, "grad_norm": 0.8077777028083801, "learning_rate": 8.100214524900103e-05, "loss": 0.022250276058912277, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.0225, "step": 174, "tokens/total": 5630016, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 79030} +{"epoch": 0.68359375, "grad_norm": 0.718685507774353, "learning_rate": 8.076468193435301e-05, "loss": 0.018044525757431984, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.01821, "step": 175, "tokens/total": 5659656, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 79454} +{"epoch": 0.6875, "grad_norm": 0.584427535533905, "learning_rate": 8.052614644612253e-05, "loss": 0.008653268218040466, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00869, "step": 176, "tokens/total": 5692408, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 79915} +{"epoch": 0.69140625, "grad_norm": 0.5644744634628296, "learning_rate": 8.028654871074489e-05, "loss": 0.005817128345370293, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00583, "step": 177, "tokens/total": 5724672, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 80356} +{"epoch": 0.6953125, "grad_norm": 0.35768353939056396, "learning_rate": 8.004589869885986e-05, "loss": 0.01321007777005434, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.0133, "step": 178, "tokens/total": 5754384, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 80774} +{"epoch": 0.69921875, "grad_norm": 0.35749390721321106, "learning_rate": 7.980420642489674e-05, "loss": 0.00920372549444437, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00925, "step": 179, "tokens/total": 5789344, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 81247} +{"epoch": 0.703125, "grad_norm": 0.30269569158554077, "learning_rate": 7.95614819466576e-05, "loss": 0.008958015590906143, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.009, "step": 180, "tokens/total": 5821816, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 81713} +{"epoch": 0.70703125, "grad_norm": 0.2554868161678314, "learning_rate": 7.931773536489872e-05, "loss": 0.0078275790438056, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00786, "step": 181, "tokens/total": 5854168, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 82158} +{"epoch": 0.7109375, "grad_norm": 0.390348345041275, "learning_rate": 7.907297682291035e-05, "loss": 0.010399932973086834, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01045, "step": 182, "tokens/total": 5886064, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 82621} +{"epoch": 0.71484375, "grad_norm": 0.23957832157611847, "learning_rate": 7.882721650609442e-05, "loss": 0.008703039959073067, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00874, "step": 183, "tokens/total": 5921648, "tokens/train_per_sec_per_gpu": 6.79, "tokens/trainable": 83122} +{"epoch": 0.71875, "grad_norm": 0.2598944902420044, "learning_rate": 7.85804646415409e-05, "loss": 0.0077856723219156265, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.00782, "step": 184, "tokens/total": 5954952, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 83578} +{"epoch": 0.72265625, "grad_norm": 0.16016454994678497, "learning_rate": 7.833273149760207e-05, "loss": 0.0009984528878703713, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.001, "step": 185, "tokens/total": 5985432, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 84048} +{"epoch": 0.7265625, "grad_norm": 0.8750929832458496, "learning_rate": 7.808402738346527e-05, "loss": 0.014847930520772934, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01496, "step": 186, "tokens/total": 6020664, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 84497} +{"epoch": 0.73046875, "grad_norm": 0.30804258584976196, "learning_rate": 7.783436264872382e-05, "loss": 0.005064752418547869, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00508, "step": 187, "tokens/total": 6055024, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 84983} +{"epoch": 0.734375, "grad_norm": 0.36235666275024414, "learning_rate": 7.758374768294647e-05, "loss": 0.008889511227607727, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00893, "step": 188, "tokens/total": 6088296, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 85462} +{"epoch": 0.73828125, "grad_norm": 0.4002234637737274, "learning_rate": 7.733219291524489e-05, "loss": 0.004737970419228077, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00475, "step": 189, "tokens/total": 6119840, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 85892} +{"epoch": 0.7421875, "grad_norm": 0.24247954785823822, "learning_rate": 7.707970881383977e-05, "loss": 0.0016467751702293754, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.00165, "step": 190, "tokens/total": 6149880, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 86341} +{"epoch": 0.74609375, "grad_norm": 1.0711240768432617, "learning_rate": 7.682630588562518e-05, "loss": 0.02559230476617813, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.02592, "step": 191, "tokens/total": 6182904, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 86772} +{"epoch": 0.75, "grad_norm": 0.23387040197849274, "learning_rate": 7.657199467573129e-05, "loss": 0.007486950606107712, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00752, "step": 192, "tokens/total": 6214144, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 87244} +{"epoch": 0.75390625, "grad_norm": 3.762739658355713, "learning_rate": 7.631678576708561e-05, "loss": 0.006447416264563799, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00647, "step": 193, "tokens/total": 6247072, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 87736} +{"epoch": 0.7578125, "grad_norm": 0.04631058871746063, "learning_rate": 7.606068977997255e-05, "loss": 0.0003396676038391888, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00034, "step": 194, "tokens/total": 6281112, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 88214} +{"epoch": 0.76171875, "grad_norm": 0.20368896424770355, "learning_rate": 7.580371737159148e-05, "loss": 0.0015097239520400763, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00151, "step": 195, "tokens/total": 6310200, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 88625} +{"epoch": 0.765625, "grad_norm": 0.07536255568265915, "learning_rate": 7.554587923561324e-05, "loss": 0.0011606537736952305, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00116, "step": 196, "tokens/total": 6341792, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 89063} +{"epoch": 0.76953125, "grad_norm": 0.3378998041152954, "learning_rate": 7.528718610173511e-05, "loss": 0.0022870053071528673, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.00229, "step": 197, "tokens/total": 6374040, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 89484} +{"epoch": 0.7734375, "grad_norm": 0.06393314152956009, "learning_rate": 7.502764873523431e-05, "loss": 0.0011202666210010648, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00112, "step": 198, "tokens/total": 6405552, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 89922} +{"epoch": 0.77734375, "grad_norm": 0.2370106428861618, "learning_rate": 7.476727793652011e-05, "loss": 0.003149718511849642, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00315, "step": 199, "tokens/total": 6436016, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 90401} +{"epoch": 0.78125, "grad_norm": 0.19121527671813965, "learning_rate": 7.450608454068415e-05, "loss": 0.0017713906709104776, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00177, "step": 200, "tokens/total": 6468656, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 90842} +{"epoch": 0.78515625, "grad_norm": 0.20390355587005615, "learning_rate": 7.424407941704987e-05, "loss": 0.003728313837200403, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00374, "step": 201, "tokens/total": 6502728, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 91343} +{"epoch": 0.7890625, "grad_norm": 0.39324310421943665, "learning_rate": 7.398127346871986e-05, "loss": 0.015165625140070915, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.01528, "step": 202, "tokens/total": 6535616, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 91798} +{"epoch": 0.79296875, "grad_norm": 0.3203586935997009, "learning_rate": 7.371767763212238e-05, "loss": 0.0006697406060993671, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00067, "step": 203, "tokens/total": 6567088, "tokens/train_per_sec_per_gpu": 10.17, "tokens/trainable": 92251} +{"epoch": 0.796875, "grad_norm": 0.14179101586341858, "learning_rate": 7.345330287655617e-05, "loss": 0.0015766527503728867, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00158, "step": 204, "tokens/total": 6600808, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 92714} +{"epoch": 0.80078125, "grad_norm": 0.3456535041332245, "learning_rate": 7.31881602037339e-05, "loss": 0.002229288686066866, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.00223, "step": 205, "tokens/total": 6631384, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 93154} +{"epoch": 0.8046875, "grad_norm": 0.10317336022853851, "learning_rate": 7.29222606473245e-05, "loss": 0.001138071296736598, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00114, "step": 206, "tokens/total": 6661888, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 93599} +{"epoch": 0.80859375, "grad_norm": 0.029759354889392853, "learning_rate": 7.265561527249383e-05, "loss": 0.00022925951634533703, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00023, "step": 207, "tokens/total": 6693456, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 94036} +{"epoch": 0.8125, "grad_norm": 0.0832960456609726, "learning_rate": 7.238823517544436e-05, "loss": 0.0009778348030522466, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00098, "step": 208, "tokens/total": 6727840, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 94506} +{"epoch": 0.81640625, "grad_norm": 0.3665315806865692, "learning_rate": 7.212013148295333e-05, "loss": 0.0022769668139517307, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00228, "step": 209, "tokens/total": 6761704, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 94988} +{"epoch": 0.8203125, "grad_norm": 0.005482575856149197, "learning_rate": 7.185131535190975e-05, "loss": 6.669942376902327e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00007, "step": 210, "tokens/total": 6792648, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 95497} +{"epoch": 0.82421875, "grad_norm": 0.04279656335711479, "learning_rate": 7.158179796885005e-05, "loss": 0.00027653819415718317, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00028, "step": 211, "tokens/total": 6824424, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 95948} +{"epoch": 0.828125, "grad_norm": 0.5132131576538086, "learning_rate": 7.131159054949273e-05, "loss": 0.002520111622288823, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00252, "step": 212, "tokens/total": 6861216, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 96422} +{"epoch": 0.83203125, "grad_norm": 0.1004396304488182, "learning_rate": 7.104070433827139e-05, "loss": 0.0008728246903046966, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00087, "step": 213, "tokens/total": 6892704, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 96821} +{"epoch": 0.8359375, "grad_norm": 0.06434912234544754, "learning_rate": 7.076915060786705e-05, "loss": 0.0006044271867722273, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.0006, "step": 214, "tokens/total": 6924816, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 97263} +{"epoch": 0.83984375, "grad_norm": 0.0958305075764656, "learning_rate": 7.049694065873882e-05, "loss": 0.0006429903442040086, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00064, "step": 215, "tokens/total": 6957536, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 97720} +{"epoch": 0.84375, "grad_norm": 0.7170233726501465, "learning_rate": 7.022408581865382e-05, "loss": 0.010559177957475185, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.01062, "step": 216, "tokens/total": 6991448, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 98194} +{"epoch": 0.84765625, "grad_norm": 0.020264416933059692, "learning_rate": 6.99505974422157e-05, "loss": 0.00015424737648572773, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00015, "step": 217, "tokens/total": 7023712, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 98667} +{"epoch": 0.8515625, "grad_norm": 0.3604339361190796, "learning_rate": 6.967648691039213e-05, "loss": 0.003011840395629406, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00302, "step": 218, "tokens/total": 7055040, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 99100} +{"epoch": 0.85546875, "grad_norm": 0.5387814044952393, "learning_rate": 6.940176563004123e-05, "loss": 0.002522213151678443, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00253, "step": 219, "tokens/total": 7084944, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 99532} +{"epoch": 0.859375, "grad_norm": 0.13106009364128113, "learning_rate": 6.912644503343682e-05, "loss": 0.0010199948446825147, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00102, "step": 220, "tokens/total": 7119264, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 99970} +{"epoch": 0.86328125, "grad_norm": 0.09447196125984192, "learning_rate": 6.885053657779273e-05, "loss": 0.00040296779479831457, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.0004, "step": 221, "tokens/total": 7150848, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 100442} +{"epoch": 0.8671875, "grad_norm": 0.03067529946565628, "learning_rate": 6.857405174478604e-05, "loss": 0.00028668102459050715, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00029, "step": 222, "tokens/total": 7185176, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 100895} +{"epoch": 0.87109375, "grad_norm": 0.4626061022281647, "learning_rate": 6.82970020400792e-05, "loss": 0.005794919561594725, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00581, "step": 223, "tokens/total": 7219024, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 101342} +{"epoch": 0.875, "grad_norm": 5.815750598907471, "learning_rate": 6.801939899284132e-05, "loss": 0.007445270195603371, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00747, "step": 224, "tokens/total": 7253816, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 101848} +{"epoch": 0.87890625, "grad_norm": 0.3828267753124237, "learning_rate": 6.774125415526827e-05, "loss": 0.00435247179120779, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00436, "step": 225, "tokens/total": 7286344, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 102319} +{"epoch": 0.8828125, "grad_norm": 0.2604454755783081, "learning_rate": 6.746257910210214e-05, "loss": 0.0024589085951447487, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00246, "step": 226, "tokens/total": 7317952, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 102769} +{"epoch": 0.88671875, "grad_norm": 0.1053524985909462, "learning_rate": 6.718338543014937e-05, "loss": 0.0018279445357620716, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00183, "step": 227, "tokens/total": 7347336, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 103219} +{"epoch": 0.890625, "grad_norm": 1.014298677444458, "learning_rate": 6.69036847577983e-05, "loss": 0.008997047320008278, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00904, "step": 228, "tokens/total": 7377592, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 103675} +{"epoch": 0.89453125, "grad_norm": 0.39947184920310974, "learning_rate": 6.662348872453553e-05, "loss": 0.0072074634954333305, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00723, "step": 229, "tokens/total": 7410184, "tokens/train_per_sec_per_gpu": 10.6, "tokens/trainable": 104136} +{"epoch": 0.8984375, "grad_norm": 0.0803765282034874, "learning_rate": 6.63428089904618e-05, "loss": 0.0013253201032057405, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00133, "step": 230, "tokens/total": 7443984, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 104588} +{"epoch": 0.90234375, "grad_norm": 0.04749743267893791, "learning_rate": 6.60616572358065e-05, "loss": 0.00014888816804159433, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00015, "step": 231, "tokens/total": 7472480, "tokens/train_per_sec_per_gpu": 10.01, "tokens/trainable": 104988} +{"epoch": 0.90625, "grad_norm": 0.9365471601486206, "learning_rate": 6.578004516044172e-05, "loss": 0.0017174372915178537, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00172, "step": 232, "tokens/total": 7505992, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 105455} +{"epoch": 0.91015625, "grad_norm": 0.10812926292419434, "learning_rate": 6.549798448339548e-05, "loss": 0.001193846226669848, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00119, "step": 233, "tokens/total": 7541168, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 105918} +{"epoch": 0.9140625, "grad_norm": 0.26391056180000305, "learning_rate": 6.521548694236384e-05, "loss": 0.0007535564946010709, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00075, "step": 234, "tokens/total": 7574208, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 106355} +{"epoch": 0.91796875, "grad_norm": 0.5046702027320862, "learning_rate": 6.493256429322259e-05, "loss": 0.01315890159457922, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.01325, "step": 235, "tokens/total": 7607016, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 106786} +{"epoch": 0.921875, "grad_norm": 0.03492911905050278, "learning_rate": 6.464922830953799e-05, "loss": 0.0003489589144010097, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00035, "step": 236, "tokens/total": 7641744, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 107219} +{"epoch": 0.92578125, "grad_norm": 0.4301534593105316, "learning_rate": 6.436549078207688e-05, "loss": 0.006682786159217358, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00671, "step": 237, "tokens/total": 7673096, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 107679} +{"epoch": 0.9296875, "grad_norm": 0.7257076501846313, "learning_rate": 6.408136351831592e-05, "loss": 0.013084979727864265, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.01317, "step": 238, "tokens/total": 7705344, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 108119} +{"epoch": 0.93359375, "grad_norm": 0.16139203310012817, "learning_rate": 6.379685834195036e-05, "loss": 0.0023884382098913193, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00239, "step": 239, "tokens/total": 7740240, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 108626} +{"epoch": 0.9375, "grad_norm": 0.16530358791351318, "learning_rate": 6.351198709240186e-05, "loss": 0.0006491647218354046, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00065, "step": 240, "tokens/total": 7775392, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 109079} +{"epoch": 0.94140625, "grad_norm": 0.33039402961730957, "learning_rate": 6.32267616243259e-05, "loss": 0.0017340773483738303, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00174, "step": 241, "tokens/total": 7807296, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 109542} +{"epoch": 0.9453125, "grad_norm": 0.07812243700027466, "learning_rate": 6.294119380711849e-05, "loss": 0.0010449547553434968, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00105, "step": 242, "tokens/total": 7838200, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 109969} +{"epoch": 0.94921875, "grad_norm": 0.698808491230011, "learning_rate": 6.265529552442209e-05, "loss": 0.015311149880290031, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01543, "step": 243, "tokens/total": 7870936, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 110431} +{"epoch": 0.953125, "grad_norm": 0.057332392781972885, "learning_rate": 6.236907867363127e-05, "loss": 0.0005850985180586576, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00059, "step": 244, "tokens/total": 7902512, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 110877} +{"epoch": 0.95703125, "grad_norm": 0.28301408886909485, "learning_rate": 6.208255516539749e-05, "loss": 0.002063036896288395, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00207, "step": 245, "tokens/total": 7933224, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 111350} +{"epoch": 0.9609375, "grad_norm": 0.32905420660972595, "learning_rate": 6.179573692313344e-05, "loss": 0.0041694650426507, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00418, "step": 246, "tokens/total": 7968456, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 111798} +{"epoch": 0.96484375, "grad_norm": 0.4185897707939148, "learning_rate": 6.150863588251694e-05, "loss": 0.005306601524353027, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00532, "step": 247, "tokens/total": 8003592, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 112264} +{"epoch": 0.96875, "grad_norm": 0.46115389466285706, "learning_rate": 6.122126399099419e-05, "loss": 0.004440301563590765, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00445, "step": 248, "tokens/total": 8040048, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 112772} +{"epoch": 0.97265625, "grad_norm": 0.8104432821273804, "learning_rate": 6.0933633207282615e-05, "loss": 0.0030875264201313257, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00309, "step": 249, "tokens/total": 8074000, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 113278} +{"epoch": 0.9765625, "grad_norm": 0.1796548068523407, "learning_rate": 6.064575550087316e-05, "loss": 0.002688421169295907, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00269, "step": 250, "tokens/total": 8108304, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 113716} +{"epoch": 0.98046875, "grad_norm": 0.27171823382377625, "learning_rate": 6.0357642851532245e-05, "loss": 0.0032277991995215416, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00323, "step": 251, "tokens/total": 8143464, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 114207} +{"epoch": 0.984375, "grad_norm": 0.5040669441223145, "learning_rate": 6.0069307248803294e-05, "loss": 0.00804364774376154, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00808, "step": 252, "tokens/total": 8175576, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 114628} +{"epoch": 0.98828125, "grad_norm": 0.10501889884471893, "learning_rate": 5.9780760691507635e-05, "loss": 0.00015300378436222672, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00015, "step": 253, "tokens/total": 8205528, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 115060} +{"epoch": 0.9921875, "grad_norm": 0.5764888525009155, "learning_rate": 5.9492015187245334e-05, "loss": 0.004174237139523029, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00418, "step": 254, "tokens/total": 8239424, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 115514} +{"epoch": 0.99609375, "grad_norm": 0.43474945425987244, "learning_rate": 5.920308275189541e-05, "loss": 0.0032240631990134716, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00323, "step": 255, "tokens/total": 8270392, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 115915} +{"epoch": 1.0, "grad_norm": 0.13866755366325378, "learning_rate": 5.8913975409115874e-05, "loss": 0.002961551770567894, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00297, "step": 256, "tokens/total": 8302096, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 116378} +{"epoch": 1.00390625, "grad_norm": 0.4461711645126343, "learning_rate": 5.8624705189843395e-05, "loss": 0.010643400251865387, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.0107, "step": 257, "tokens/total": 8337064, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 116865} +{"epoch": 1.0078125, "grad_norm": 0.2159780114889145, "learning_rate": 5.833528413179249e-05, "loss": 0.0011617698473855853, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00116, "step": 258, "tokens/total": 8372840, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 117363} +{"epoch": 1.01171875, "grad_norm": 0.10402460396289825, "learning_rate": 5.80457242789548e-05, "loss": 0.001483182655647397, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00148, "step": 259, "tokens/total": 8409728, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 117860} +{"epoch": 1.015625, "grad_norm": 0.3103790879249573, "learning_rate": 5.77560376810977e-05, "loss": 0.0036294618621468544, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.00364, "step": 260, "tokens/total": 8442296, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 118324} +{"epoch": 1.01953125, "grad_norm": 0.15490856766700745, "learning_rate": 5.7466236393263005e-05, "loss": 0.0027101049199700356, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00271, "step": 261, "tokens/total": 8474568, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 118770} +{"epoch": 1.0234375, "grad_norm": 0.049445949494838715, "learning_rate": 5.717633247526522e-05, "loss": 0.0005567588377743959, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00056, "step": 262, "tokens/total": 8505520, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 119206} +{"epoch": 1.02734375, "grad_norm": 0.025037869811058044, "learning_rate": 5.688633799118971e-05, "loss": 0.0002567955816630274, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00026, "step": 263, "tokens/total": 8535064, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 119610} +{"epoch": 1.03125, "grad_norm": 0.03797755390405655, "learning_rate": 5.659626500889066e-05, "loss": 0.0004601108084898442, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00046, "step": 264, "tokens/total": 8563600, "tokens/train_per_sec_per_gpu": 7.09, "tokens/trainable": 120080} +{"epoch": 1.03515625, "grad_norm": 0.3568636476993561, "learning_rate": 5.6306125599488905e-05, "loss": 0.005535652860999107, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00555, "step": 265, "tokens/total": 8592648, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 120519} +{"epoch": 1.0390625, "grad_norm": 0.05474524199962616, "learning_rate": 5.601593183686955e-05, "loss": 0.00051836296916008, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00052, "step": 266, "tokens/total": 8627328, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 120995} +{"epoch": 1.04296875, "grad_norm": 0.19349491596221924, "learning_rate": 5.572569579717961e-05, "loss": 0.002732088090851903, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00274, "step": 267, "tokens/total": 8659760, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 121459} +{"epoch": 1.046875, "grad_norm": 0.007730253506451845, "learning_rate": 5.543542955832538e-05, "loss": 9.865823085419834e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.0001, "step": 268, "tokens/total": 8696968, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 121902} +{"epoch": 1.05078125, "grad_norm": 0.08598754554986954, "learning_rate": 5.514514519946986e-05, "loss": 0.0011416103225201368, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00114, "step": 269, "tokens/total": 8735392, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 122399} +{"epoch": 1.0546875, "grad_norm": 0.035149432718753815, "learning_rate": 5.485485480053015e-05, "loss": 0.0002857904473785311, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00029, "step": 270, "tokens/total": 8769840, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 122804} +{"epoch": 1.05859375, "grad_norm": 0.03338836133480072, "learning_rate": 5.4564570441674645e-05, "loss": 0.00010565879347268492, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00011, "step": 271, "tokens/total": 8800648, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 123244} +{"epoch": 1.0625, "grad_norm": 0.0019740283023566008, "learning_rate": 5.42743042028204e-05, "loss": 3.205867687938735e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00003, "step": 272, "tokens/total": 8832480, "tokens/train_per_sec_per_gpu": 6.84, "tokens/trainable": 123683} +{"epoch": 1.06640625, "grad_norm": 0.08643896132707596, "learning_rate": 5.3984068163130464e-05, "loss": 0.0008074198267422616, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00081, "step": 273, "tokens/total": 8864840, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 124153} +{"epoch": 1.0703125, "grad_norm": 0.2160847783088684, "learning_rate": 5.369387440051111e-05, "loss": 0.0030535964760929346, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00306, "step": 274, "tokens/total": 8897272, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 124616} +{"epoch": 1.07421875, "grad_norm": 0.12607334554195404, "learning_rate": 5.340373499110935e-05, "loss": 0.0003846338950097561, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00038, "step": 275, "tokens/total": 8930688, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 125109} +{"epoch": 1.078125, "grad_norm": 0.0015076157869771123, "learning_rate": 5.3113662008810304e-05, "loss": 2.1120566088939086e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00002, "step": 276, "tokens/total": 8967216, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 125591} +{"epoch": 1.08203125, "grad_norm": 0.1746973693370819, "learning_rate": 5.282366752473479e-05, "loss": 0.0009688625577837229, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00097, "step": 277, "tokens/total": 9001368, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 126065} +{"epoch": 1.0859375, "grad_norm": 0.015611347742378712, "learning_rate": 5.2533763606737005e-05, "loss": 0.00010646578448358923, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00011, "step": 278, "tokens/total": 9036808, "tokens/train_per_sec_per_gpu": 6.93, "tokens/trainable": 126480} +{"epoch": 1.08984375, "grad_norm": 0.01392409298568964, "learning_rate": 5.224396231890232e-05, "loss": 0.0001099322471418418, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00011, "step": 279, "tokens/total": 9067840, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 126929} +{"epoch": 1.09375, "grad_norm": 0.5592253804206848, "learning_rate": 5.195427572104522e-05, "loss": 0.015301416628062725, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.01542, "step": 280, "tokens/total": 9098936, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 127366} +{"epoch": 1.09765625, "grad_norm": 0.0058529856614768505, "learning_rate": 5.166471586820751e-05, "loss": 5.5999975302256644e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00006, "step": 281, "tokens/total": 9133264, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 127831} +{"epoch": 1.1015625, "grad_norm": 0.010115265846252441, "learning_rate": 5.1375294810156615e-05, "loss": 7.231541530927643e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00007, "step": 282, "tokens/total": 9165008, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 128278} +{"epoch": 1.10546875, "grad_norm": 0.09704276919364929, "learning_rate": 5.1086024590884144e-05, "loss": 0.00044296577107161283, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00044, "step": 283, "tokens/total": 9197952, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 128727} +{"epoch": 1.109375, "grad_norm": 0.4147341251373291, "learning_rate": 5.079691724810461e-05, "loss": 0.0013574741315096617, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00136, "step": 284, "tokens/total": 9232576, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 129162} +{"epoch": 1.11328125, "grad_norm": 0.027643555775284767, "learning_rate": 5.0507984812754684e-05, "loss": 0.00016950782446656376, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00017, "step": 285, "tokens/total": 9264568, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 129659} +{"epoch": 1.1171875, "grad_norm": 0.04004650563001633, "learning_rate": 5.021923930849237e-05, "loss": 0.00022094279120210558, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00022, "step": 286, "tokens/total": 9296544, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 130145} +{"epoch": 1.12109375, "grad_norm": 0.0008340429631061852, "learning_rate": 4.99306927511967e-05, "loss": 1.1548805559868924e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00001, "step": 287, "tokens/total": 9327936, "tokens/train_per_sec_per_gpu": 10.5, "tokens/trainable": 130585} +{"epoch": 1.125, "grad_norm": 0.22382839024066925, "learning_rate": 4.964235714846775e-05, "loss": 0.0009153201244771481, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00092, "step": 288, "tokens/total": 9357984, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 131005} +{"epoch": 1.12890625, "grad_norm": 0.023247206583619118, "learning_rate": 4.9354244499126866e-05, "loss": 0.0001603159325895831, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00016, "step": 289, "tokens/total": 9388664, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 131471} +{"epoch": 1.1328125, "grad_norm": 0.16754476726055145, "learning_rate": 4.90663667927174e-05, "loss": 0.0011593119706958532, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00116, "step": 290, "tokens/total": 9420504, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 131920} +{"epoch": 1.13671875, "grad_norm": 0.0020202971063554287, "learning_rate": 4.877873600900581e-05, "loss": 2.835889972629957e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00003, "step": 291, "tokens/total": 9452664, "tokens/train_per_sec_per_gpu": 6.34, "tokens/trainable": 132379} +{"epoch": 1.140625, "grad_norm": 0.004856865853071213, "learning_rate": 4.849136411748306e-05, "loss": 6.492385728051886e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00006, "step": 292, "tokens/total": 9488896, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 132851} +{"epoch": 1.14453125, "grad_norm": 0.7534581422805786, "learning_rate": 4.8204263076866574e-05, "loss": 0.012913145124912262, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.013, "step": 293, "tokens/total": 9522112, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 133351} +{"epoch": 1.1484375, "grad_norm": 0.009672252461314201, "learning_rate": 4.791744483460251e-05, "loss": 5.567634798353538e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00006, "step": 294, "tokens/total": 9555632, "tokens/train_per_sec_per_gpu": 6.65, "tokens/trainable": 133784} +{"epoch": 1.15234375, "grad_norm": 0.04718039929866791, "learning_rate": 4.7630921326368736e-05, "loss": 0.0001484099484514445, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00015, "step": 295, "tokens/total": 9590904, "tokens/train_per_sec_per_gpu": 6.46, "tokens/trainable": 134209} +{"epoch": 1.15625, "grad_norm": 0.06154409795999527, "learning_rate": 4.7344704475577916e-05, "loss": 0.00045448067248798907, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00045, "step": 296, "tokens/total": 9620288, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 134645} +{"epoch": 1.16015625, "grad_norm": 0.15832051634788513, "learning_rate": 4.705880619288153e-05, "loss": 0.0020587327890098095, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00206, "step": 297, "tokens/total": 9651984, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 135047} +{"epoch": 1.1640625, "grad_norm": 1.4507588148117065, "learning_rate": 4.677323837567412e-05, "loss": 0.007110184524208307, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00714, "step": 298, "tokens/total": 9681368, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 135474} +{"epoch": 1.16796875, "grad_norm": 0.09572083503007889, "learning_rate": 4.6488012907598146e-05, "loss": 0.0003969114040955901, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.0004, "step": 299, "tokens/total": 9712872, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 135948} +{"epoch": 1.171875, "grad_norm": 0.5708215832710266, "learning_rate": 4.620314165804964e-05, "loss": 0.007049119099974632, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00707, "step": 300, "tokens/total": 9745184, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 136421} +{"epoch": 1.17578125, "grad_norm": 0.02900431491434574, "learning_rate": 4.591863648168407e-05, "loss": 0.00025898791500367224, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00026, "step": 301, "tokens/total": 9778928, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 136886} +{"epoch": 1.1796875, "grad_norm": 0.004458446521311998, "learning_rate": 4.5634509217923135e-05, "loss": 5.26636686117854e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00005, "step": 302, "tokens/total": 9810752, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 137321} +{"epoch": 1.18359375, "grad_norm": 0.10162343829870224, "learning_rate": 4.535077169046201e-05, "loss": 0.0006521859904751182, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00065, "step": 303, "tokens/total": 9843672, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 137758} +{"epoch": 1.1875, "grad_norm": 0.5490950345993042, "learning_rate": 4.506743570677743e-05, "loss": 0.005628087557852268, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00564, "step": 304, "tokens/total": 9873752, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 138190} +{"epoch": 1.19140625, "grad_norm": 0.034356776624917984, "learning_rate": 4.478451305763618e-05, "loss": 0.0002599115832708776, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00026, "step": 305, "tokens/total": 9906272, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 138650} +{"epoch": 1.1953125, "grad_norm": 0.2595946788787842, "learning_rate": 4.450201551660454e-05, "loss": 0.002663220977410674, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00267, "step": 306, "tokens/total": 9936296, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 139082} +{"epoch": 1.19921875, "grad_norm": 0.1195145770907402, "learning_rate": 4.4219954839558276e-05, "loss": 0.0008606099872849882, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00086, "step": 307, "tokens/total": 9967720, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 139523} +{"epoch": 1.203125, "grad_norm": 0.428059846162796, "learning_rate": 4.393834276419352e-05, "loss": 0.003539833938702941, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.00355, "step": 308, "tokens/total": 9997176, "tokens/train_per_sec_per_gpu": 10.88, "tokens/trainable": 140010} +{"epoch": 1.20703125, "grad_norm": 0.3060550391674042, "learning_rate": 4.36571910095382e-05, "loss": 0.0006546213990077376, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00065, "step": 309, "tokens/total": 10031872, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 140454} +{"epoch": 1.2109375, "grad_norm": 0.3332705795764923, "learning_rate": 4.337651127546448e-05, "loss": 0.0029440594371408224, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00295, "step": 310, "tokens/total": 10065240, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 140905} +{"epoch": 1.21484375, "grad_norm": 0.12650257349014282, "learning_rate": 4.3096315242201736e-05, "loss": 0.0009799260878935456, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00098, "step": 311, "tokens/total": 10096640, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 141339} +{"epoch": 1.21875, "grad_norm": 0.2922199070453644, "learning_rate": 4.2816614569850635e-05, "loss": 0.0007771336822770536, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00078, "step": 312, "tokens/total": 10130192, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 141778} +{"epoch": 1.22265625, "grad_norm": 0.2793675363063812, "learning_rate": 4.2537420897897864e-05, "loss": 0.002903370186686516, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00291, "step": 313, "tokens/total": 10164248, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 142255} +{"epoch": 1.2265625, "grad_norm": 0.03735999017953873, "learning_rate": 4.225874584473174e-05, "loss": 0.00044943869579583406, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00045, "step": 314, "tokens/total": 10194664, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 142693} +{"epoch": 1.23046875, "grad_norm": 0.018737494945526123, "learning_rate": 4.19806010071587e-05, "loss": 0.0001190364346257411, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00012, "step": 315, "tokens/total": 10227768, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 143157} +{"epoch": 1.234375, "grad_norm": 0.012396497651934624, "learning_rate": 4.170299795992081e-05, "loss": 0.00012055510887876153, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00012, "step": 316, "tokens/total": 10260464, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 143603} +{"epoch": 1.23828125, "grad_norm": 0.07563463598489761, "learning_rate": 4.142594825521398e-05, "loss": 0.0006353251519612968, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00064, "step": 317, "tokens/total": 10297528, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 144084} +{"epoch": 1.2421875, "grad_norm": 0.0015867743641138077, "learning_rate": 4.114946342220728e-05, "loss": 1.4271968211687636e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00001, "step": 318, "tokens/total": 10331448, "tokens/train_per_sec_per_gpu": 7.04, "tokens/trainable": 144532} +{"epoch": 1.24609375, "grad_norm": 0.002135329879820347, "learning_rate": 4.087355496656321e-05, "loss": 3.131314952042885e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00003, "step": 319, "tokens/total": 10363152, "tokens/train_per_sec_per_gpu": 10.24, "tokens/trainable": 145004} +{"epoch": 1.25, "grad_norm": 0.19845908880233765, "learning_rate": 4.05982343699588e-05, "loss": 0.000628929934464395, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00063, "step": 320, "tokens/total": 10397832, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 145498} +{"epoch": 1.25390625, "grad_norm": 0.018875746056437492, "learning_rate": 4.0323513089607876e-05, "loss": 5.726820381823927e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00006, "step": 321, "tokens/total": 10431416, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 145965} +{"epoch": 1.2578125, "grad_norm": 0.5865472555160522, "learning_rate": 4.004940255778431e-05, "loss": 0.004501592367887497, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00451, "step": 322, "tokens/total": 10467384, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 146409} +{"epoch": 1.26171875, "grad_norm": 0.0034782097209244967, "learning_rate": 3.977591418134619e-05, "loss": 2.7128464353154413e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00003, "step": 323, "tokens/total": 10500792, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 146849} +{"epoch": 1.265625, "grad_norm": 0.20277705788612366, "learning_rate": 3.95030593412612e-05, "loss": 0.001779767102561891, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00178, "step": 324, "tokens/total": 10532592, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 147272} +{"epoch": 1.26953125, "grad_norm": 0.0008200973970815539, "learning_rate": 3.923084939213296e-05, "loss": 1.1018444638466462e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00001, "step": 325, "tokens/total": 10562392, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 147694} +{"epoch": 1.2734375, "grad_norm": 0.43600714206695557, "learning_rate": 3.895929566172861e-05, "loss": 0.0015872560907155275, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00159, "step": 326, "tokens/total": 10595416, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 148158} +{"epoch": 1.27734375, "grad_norm": 0.017886919900774956, "learning_rate": 3.868840945050728e-05, "loss": 0.00016162765678018332, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00016, "step": 327, "tokens/total": 10630504, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 148590} +{"epoch": 1.28125, "grad_norm": 0.5240402221679688, "learning_rate": 3.841820203114995e-05, "loss": 0.003609459148719907, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00362, "step": 328, "tokens/total": 10661632, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 149037} +{"epoch": 1.28515625, "grad_norm": 0.0012612921418622136, "learning_rate": 3.814868464809027e-05, "loss": 1.3396055692282971e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00001, "step": 329, "tokens/total": 10691544, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 149439} +{"epoch": 1.2890625, "grad_norm": 0.00972528476268053, "learning_rate": 3.787986851704667e-05, "loss": 7.263562292791903e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00007, "step": 330, "tokens/total": 10724768, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 149869} +{"epoch": 1.29296875, "grad_norm": 0.0003936458670068532, "learning_rate": 3.7611764824555654e-05, "loss": 5.870962468179641e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00001, "step": 331, "tokens/total": 10756008, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 150343} +{"epoch": 1.296875, "grad_norm": 0.0014341867063194513, "learning_rate": 3.734438472750619e-05, "loss": 1.2498849173425697e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00001, "step": 332, "tokens/total": 10793152, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 150814} +{"epoch": 1.30078125, "grad_norm": 0.16176892817020416, "learning_rate": 3.707773935267552e-05, "loss": 0.0008792415028437972, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00088, "step": 333, "tokens/total": 10827624, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 151300} +{"epoch": 1.3046875, "grad_norm": 0.00022062240168452263, "learning_rate": 3.68118397962661e-05, "loss": 4.881990207650233e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.0, "step": 334, "tokens/total": 10859712, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 151708} +{"epoch": 1.30859375, "grad_norm": 0.042793285101652145, "learning_rate": 3.654669712344384e-05, "loss": 8.108992915367708e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00008, "step": 335, "tokens/total": 10888792, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 152164} +{"epoch": 1.3125, "grad_norm": 0.0008495181100443006, "learning_rate": 3.628232236787763e-05, "loss": 5.665421213052468e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00001, "step": 336, "tokens/total": 10922968, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 152603} +{"epoch": 1.31640625, "grad_norm": 0.00026047570281662047, "learning_rate": 3.6018726531280144e-05, "loss": 5.757371582149062e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00001, "step": 337, "tokens/total": 10958000, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 153094} +{"epoch": 1.3203125, "grad_norm": 0.34993159770965576, "learning_rate": 3.575592058295017e-05, "loss": 0.027525797486305237, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.02791, "step": 338, "tokens/total": 10989872, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 153573} +{"epoch": 1.32421875, "grad_norm": 0.00029207044281065464, "learning_rate": 3.549391545931585e-05, "loss": 3.4142153708671685e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.0, "step": 339, "tokens/total": 11020784, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 154033} +{"epoch": 1.328125, "grad_norm": 0.08888532966375351, "learning_rate": 3.5232722063479914e-05, "loss": 0.00048370141303166747, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00048, "step": 340, "tokens/total": 11053240, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 154482} +{"epoch": 1.33203125, "grad_norm": 0.08199572563171387, "learning_rate": 3.49723512647657e-05, "loss": 0.000570812146179378, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00057, "step": 341, "tokens/total": 11089040, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 154982} +{"epoch": 1.3359375, "grad_norm": 0.06257230043411255, "learning_rate": 3.471281389826491e-05, "loss": 0.0001614992943359539, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00016, "step": 342, "tokens/total": 11119672, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 155387} +{"epoch": 1.33984375, "grad_norm": 0.001359422691166401, "learning_rate": 3.4454120764386764e-05, "loss": 1.7583330190973356e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00002, "step": 343, "tokens/total": 11155040, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 155842} +{"epoch": 1.34375, "grad_norm": 0.0299766194075346, "learning_rate": 3.4196282628408526e-05, "loss": 0.00013530897558666766, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00014, "step": 344, "tokens/total": 11185952, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 156287} +{"epoch": 1.34765625, "grad_norm": 0.23160867393016815, "learning_rate": 3.3939310220027456e-05, "loss": 0.0019145102705806494, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00192, "step": 345, "tokens/total": 11219272, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 156782} +{"epoch": 1.3515625, "grad_norm": 0.0007788282819092274, "learning_rate": 3.3683214232914404e-05, "loss": 1.4409341019927524e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00001, "step": 346, "tokens/total": 11253424, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 157255} +{"epoch": 1.35546875, "grad_norm": 0.0017726224614307284, "learning_rate": 3.342800532426873e-05, "loss": 3.0399889510590583e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00003, "step": 347, "tokens/total": 11286216, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 157721} +{"epoch": 1.359375, "grad_norm": 0.0027885776944458485, "learning_rate": 3.317369411437484e-05, "loss": 3.409260170883499e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00003, "step": 348, "tokens/total": 11319480, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 158189} +{"epoch": 1.36328125, "grad_norm": 0.0037172732409089804, "learning_rate": 3.292029118616024e-05, "loss": 7.211287447717041e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00007, "step": 349, "tokens/total": 11350472, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 158612} +{"epoch": 1.3671875, "grad_norm": 0.008674347773194313, "learning_rate": 3.266780708475511e-05, "loss": 9.02972969925031e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00009, "step": 350, "tokens/total": 11383088, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 159090} +{"epoch": 1.37109375, "grad_norm": 0.14725637435913086, "learning_rate": 3.241625231705354e-05, "loss": 0.00040852322126738727, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00041, "step": 351, "tokens/total": 11418392, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 159549} +{"epoch": 1.375, "grad_norm": 0.0018361771944910288, "learning_rate": 3.216563735127618e-05, "loss": 2.4922730517573655e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00002, "step": 352, "tokens/total": 11449784, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 160026} +{"epoch": 1.37890625, "grad_norm": 0.40573984384536743, "learning_rate": 3.191597261653475e-05, "loss": 0.007753729820251465, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00778, "step": 353, "tokens/total": 11484240, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 160487} +{"epoch": 1.3828125, "grad_norm": 0.5074560046195984, "learning_rate": 3.166726850239794e-05, "loss": 0.003231610171496868, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00324, "step": 354, "tokens/total": 11513360, "tokens/train_per_sec_per_gpu": 11.09, "tokens/trainable": 160935} +{"epoch": 1.38671875, "grad_norm": 0.0019051234703511, "learning_rate": 3.141953535845912e-05, "loss": 3.3544376492500305e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00003, "step": 355, "tokens/total": 11543112, "tokens/train_per_sec_per_gpu": 10.29, "tokens/trainable": 161385} +{"epoch": 1.390625, "grad_norm": 0.014727535657584667, "learning_rate": 3.11727834939056e-05, "loss": 8.338835323229432e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00008, "step": 356, "tokens/total": 11573464, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 161849} +{"epoch": 1.39453125, "grad_norm": 0.27106964588165283, "learning_rate": 3.092702317708967e-05, "loss": 0.0064613656140863895, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00648, "step": 357, "tokens/total": 11604896, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 162306} +{"epoch": 1.3984375, "grad_norm": 0.0029736338183283806, "learning_rate": 3.0682264635101276e-05, "loss": 5.4810356232337654e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00005, "step": 358, "tokens/total": 11636368, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 162759} +{"epoch": 1.40234375, "grad_norm": 0.0027213760185986757, "learning_rate": 3.0438518053342407e-05, "loss": 2.9896144042140804e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00003, "step": 359, "tokens/total": 11670216, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 163230} +{"epoch": 1.40625, "grad_norm": 0.08016738295555115, "learning_rate": 3.0195793575103266e-05, "loss": 0.00023560399131383747, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00024, "step": 360, "tokens/total": 11707816, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 163694} +{"epoch": 1.41015625, "grad_norm": 0.33063581585884094, "learning_rate": 2.9954101301140146e-05, "loss": 0.005708711221814156, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00573, "step": 361, "tokens/total": 11742584, "tokens/train_per_sec_per_gpu": 6.96, "tokens/trainable": 164144} +{"epoch": 1.4140625, "grad_norm": 0.007371500134468079, "learning_rate": 2.9713451289255123e-05, "loss": 0.0001211246635648422, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00012, "step": 362, "tokens/total": 11770384, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 164585} +{"epoch": 1.41796875, "grad_norm": 0.014559725299477577, "learning_rate": 2.9473853553877484e-05, "loss": 0.00023456005146726966, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00023, "step": 363, "tokens/total": 11803208, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 165059} +{"epoch": 1.421875, "grad_norm": 0.0016077959444373846, "learning_rate": 2.9235318065647e-05, "loss": 2.118211159540806e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00002, "step": 364, "tokens/total": 11840480, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 165519} +{"epoch": 1.42578125, "grad_norm": 0.009024551138281822, "learning_rate": 2.8997854750998964e-05, "loss": 0.00014248481602407992, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00014, "step": 365, "tokens/total": 11875544, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 165984} +{"epoch": 1.4296875, "grad_norm": 0.03434529900550842, "learning_rate": 2.8761473491751258e-05, "loss": 0.00024170172400772572, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00024, "step": 366, "tokens/total": 11907656, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 166419} +{"epoch": 1.43359375, "grad_norm": 0.01771737076342106, "learning_rate": 2.8526184124692883e-05, "loss": 0.00016865594079717994, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00017, "step": 367, "tokens/total": 11941904, "tokens/train_per_sec_per_gpu": 6.46, "tokens/trainable": 166857} +{"epoch": 1.4375, "grad_norm": 0.6157798171043396, "learning_rate": 2.829199644117484e-05, "loss": 0.01380438543856144, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.0139, "step": 368, "tokens/total": 11973624, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 167267} +{"epoch": 1.44140625, "grad_norm": 0.9017988443374634, "learning_rate": 2.8058920186702553e-05, "loss": 0.009268082678318024, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00931, "step": 369, "tokens/total": 12010360, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 167740} +{"epoch": 1.4453125, "grad_norm": 0.005740329157561064, "learning_rate": 2.782696506053033e-05, "loss": 8.471019100397825e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00008, "step": 370, "tokens/total": 12044544, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 168172} +{"epoch": 1.44921875, "grad_norm": 0.07535494863986969, "learning_rate": 2.7596140715257824e-05, "loss": 0.0007381763425655663, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00074, "step": 371, "tokens/total": 12075792, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 168637} +{"epoch": 1.453125, "grad_norm": 0.00694861076772213, "learning_rate": 2.7366456756428184e-05, "loss": 7.973191532073542e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00008, "step": 372, "tokens/total": 12108088, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 169094} +{"epoch": 1.45703125, "grad_norm": 0.16273988783359528, "learning_rate": 2.7137922742128486e-05, "loss": 0.002759824972599745, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00276, "step": 373, "tokens/total": 12140888, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 169593} +{"epoch": 1.4609375, "grad_norm": 0.5574415326118469, "learning_rate": 2.691054818259188e-05, "loss": 0.013300751335918903, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.01339, "step": 374, "tokens/total": 12174560, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 170066} +{"epoch": 1.46484375, "grad_norm": 0.08313028514385223, "learning_rate": 2.6684342539801933e-05, "loss": 0.0005807778215967119, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00058, "step": 375, "tokens/total": 12204848, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 170518} +{"epoch": 1.46875, "grad_norm": 0.11019548773765564, "learning_rate": 2.645931522709877e-05, "loss": 0.0013237864477559924, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00132, "step": 376, "tokens/total": 12238472, "tokens/train_per_sec_per_gpu": 10.85, "tokens/trainable": 170998} +{"epoch": 1.47265625, "grad_norm": 0.17969545722007751, "learning_rate": 2.6235475608787365e-05, "loss": 0.0024813322816044092, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00248, "step": 377, "tokens/total": 12271024, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 171459} +{"epoch": 1.4765625, "grad_norm": 0.3346485495567322, "learning_rate": 2.6012832999747916e-05, "loss": 0.0040630982257425785, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00407, "step": 378, "tokens/total": 12301312, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 171905} +{"epoch": 1.48046875, "grad_norm": 0.4026930034160614, "learning_rate": 2.579139666504821e-05, "loss": 0.009503492154181004, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00955, "step": 379, "tokens/total": 12337072, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 172351} +{"epoch": 1.484375, "grad_norm": 0.018198346719145775, "learning_rate": 2.557117581955798e-05, "loss": 0.0001601915864739567, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00016, "step": 380, "tokens/total": 12368000, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 172817} +{"epoch": 1.48828125, "grad_norm": 0.5823228359222412, "learning_rate": 2.5352179627565532e-05, "loss": 0.012811425141990185, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.01289, "step": 381, "tokens/total": 12399880, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 173258} +{"epoch": 1.4921875, "grad_norm": 0.00922594964504242, "learning_rate": 2.5134417202396277e-05, "loss": 0.00019856105791404843, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.0002, "step": 382, "tokens/total": 12432288, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 173665} +{"epoch": 1.49609375, "grad_norm": 0.020497050136327744, "learning_rate": 2.491789760603361e-05, "loss": 0.0004817575681954622, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00048, "step": 383, "tokens/total": 12465592, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 174095} +{"epoch": 1.5, "grad_norm": 0.11979613453149796, "learning_rate": 2.4702629848741764e-05, "loss": 0.0016216009389609098, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00162, "step": 384, "tokens/total": 12498200, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 174550} +{"epoch": 1.50390625, "grad_norm": 0.07445727288722992, "learning_rate": 2.4488622888690785e-05, "loss": 0.001152463024482131, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00115, "step": 385, "tokens/total": 12530280, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 174990} +{"epoch": 1.5078125, "grad_norm": 0.06252644956111908, "learning_rate": 2.427588563158384e-05, "loss": 0.0012004413874819875, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.0012, "step": 386, "tokens/total": 12563360, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 175476} +{"epoch": 1.51171875, "grad_norm": 0.042557280510663986, "learning_rate": 2.406442693028651e-05, "loss": 0.0005736224120482802, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00057, "step": 387, "tokens/total": 12593032, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 175897} +{"epoch": 1.515625, "grad_norm": 0.007495563477277756, "learning_rate": 2.3854255584458547e-05, "loss": 0.00017981002747546881, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00018, "step": 388, "tokens/total": 12628272, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 176324} +{"epoch": 1.51953125, "grad_norm": 0.2809073030948639, "learning_rate": 2.3645380340187508e-05, "loss": 0.002416695933789015, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00242, "step": 389, "tokens/total": 12661368, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 176805} +{"epoch": 1.5234375, "grad_norm": 0.10065944492816925, "learning_rate": 2.3437809889624914e-05, "loss": 0.0022320859134197235, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.77, "memory/max_allocated (GiB)": 63.77, "ppl": 1.00223, "step": 390, "tokens/total": 12691168, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 177256} +{"epoch": 1.52734375, "grad_norm": 0.016981367021799088, "learning_rate": 2.3231552870624487e-05, "loss": 0.0003949041129089892, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00039, "step": 391, "tokens/total": 12721264, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 177682} +{"epoch": 1.53125, "grad_norm": 0.08321041613817215, "learning_rate": 2.3026617866382657e-05, "loss": 0.0010144161060452461, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00101, "step": 392, "tokens/total": 12753640, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 178179} +{"epoch": 1.53515625, "grad_norm": 0.046271588653326035, "learning_rate": 2.2823013405081507e-05, "loss": 0.0007580781821161509, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00076, "step": 393, "tokens/total": 12787056, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 178676} +{"epoch": 1.5390625, "grad_norm": 0.02852015197277069, "learning_rate": 2.2620747959533722e-05, "loss": 0.0003369513724464923, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00034, "step": 394, "tokens/total": 12819312, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 179173} +{"epoch": 1.54296875, "grad_norm": 0.057789940387010574, "learning_rate": 2.2419829946830123e-05, "loss": 0.0005477925296872854, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00055, "step": 395, "tokens/total": 12853376, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 179624} +{"epoch": 1.546875, "grad_norm": 0.061932098120450974, "learning_rate": 2.2220267727989325e-05, "loss": 0.0006530380924232304, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00065, "step": 396, "tokens/total": 12882960, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 180078} +{"epoch": 1.55078125, "grad_norm": 0.07771562039852142, "learning_rate": 2.202206960760984e-05, "loss": 0.0016011182451620698, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.0016, "step": 397, "tokens/total": 12917016, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 180585} +{"epoch": 1.5546875, "grad_norm": 0.049522630870342255, "learning_rate": 2.182524383352446e-05, "loss": 0.0007855637231841683, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00079, "step": 398, "tokens/total": 12950056, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 181039} +{"epoch": 1.55859375, "grad_norm": 0.1457129716873169, "learning_rate": 2.1629798596457056e-05, "loss": 0.0021584441419690847, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00216, "step": 399, "tokens/total": 12981728, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 181489} +{"epoch": 1.5625, "grad_norm": 0.07552015781402588, "learning_rate": 2.1435742029681725e-05, "loss": 0.00042250531259924173, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00042, "step": 400, "tokens/total": 13011464, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 181906} +{"epoch": 1.56640625, "grad_norm": 0.0023555676452815533, "learning_rate": 2.124308220868431e-05, "loss": 6.0013102483935654e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00006, "step": 401, "tokens/total": 13044432, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 182361} +{"epoch": 1.5703125, "grad_norm": 0.01632835529744625, "learning_rate": 2.105182715082638e-05, "loss": 0.00040837665437720716, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00041, "step": 402, "tokens/total": 13075608, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 182838} +{"epoch": 1.57421875, "grad_norm": 0.09953894466161728, "learning_rate": 2.0861984815011552e-05, "loss": 0.001527769840322435, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00153, "step": 403, "tokens/total": 13107352, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 183269} +{"epoch": 1.578125, "grad_norm": 0.031569402664899826, "learning_rate": 2.0673563101354323e-05, "loss": 0.0004181621188763529, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00042, "step": 404, "tokens/total": 13140736, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 183699} +{"epoch": 1.58203125, "grad_norm": 0.02740732580423355, "learning_rate": 2.0486569850851317e-05, "loss": 0.00020077699446119368, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.0002, "step": 405, "tokens/total": 13174040, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 184162} +{"epoch": 1.5859375, "grad_norm": 0.015682872384786606, "learning_rate": 2.0301012845054956e-05, "loss": 0.00025384739274159074, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00025, "step": 406, "tokens/total": 13207232, "tokens/train_per_sec_per_gpu": 6.65, "tokens/trainable": 184576} +{"epoch": 1.58984375, "grad_norm": 0.3073991537094116, "learning_rate": 2.011689980574966e-05, "loss": 0.004268933553248644, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00428, "step": 407, "tokens/total": 13241160, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 184995} +{"epoch": 1.59375, "grad_norm": 0.05152052640914917, "learning_rate": 1.993423839463052e-05, "loss": 0.0007983939722180367, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.0008, "step": 408, "tokens/total": 13273104, "tokens/train_per_sec_per_gpu": 10.49, "tokens/trainable": 185452} +{"epoch": 1.59765625, "grad_norm": 0.008453101851046085, "learning_rate": 1.975303621298445e-05, "loss": 0.00010478904005140066, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.0001, "step": 409, "tokens/total": 13308136, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 185904} +{"epoch": 1.6015625, "grad_norm": 0.007348158396780491, "learning_rate": 1.957330080137385e-05, "loss": 9.713362669572234e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.0001, "step": 410, "tokens/total": 13337816, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 186355} +{"epoch": 1.60546875, "grad_norm": 0.5382251739501953, "learning_rate": 1.9395039639322864e-05, "loss": 0.008642895147204399, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00868, "step": 411, "tokens/total": 13372576, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 186831} +{"epoch": 1.609375, "grad_norm": 0.0012179454788565636, "learning_rate": 1.9218260145006073e-05, "loss": 3.3541524317115545e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00003, "step": 412, "tokens/total": 13406664, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 187317} +{"epoch": 1.61328125, "grad_norm": 0.015679050236940384, "learning_rate": 1.904296967493982e-05, "loss": 0.00016272623906843364, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00016, "step": 413, "tokens/total": 13438256, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 187773} +{"epoch": 1.6171875, "grad_norm": 0.26036953926086426, "learning_rate": 1.8869175523676064e-05, "loss": 0.00032815735903568566, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00033, "step": 414, "tokens/total": 13472096, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 188268} +{"epoch": 1.62109375, "grad_norm": 0.015624702908098698, "learning_rate": 1.869688492349885e-05, "loss": 0.0002462793199811131, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00025, "step": 415, "tokens/total": 13504192, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 188720} +{"epoch": 1.625, "grad_norm": 0.19116279482841492, "learning_rate": 1.85261050441233e-05, "loss": 0.0023759163450449705, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00238, "step": 416, "tokens/total": 13537392, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 189177} +{"epoch": 1.62890625, "grad_norm": 0.24977004528045654, "learning_rate": 1.8356842992397304e-05, "loss": 0.003037955379113555, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00304, "step": 417, "tokens/total": 13569344, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 189617} +{"epoch": 1.6328125, "grad_norm": 0.2028685063123703, "learning_rate": 1.8189105812005714e-05, "loss": 0.002775578759610653, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00278, "step": 418, "tokens/total": 13602200, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 190028} +{"epoch": 1.63671875, "grad_norm": 0.1780310422182083, "learning_rate": 1.802290048317732e-05, "loss": 0.0014463576953858137, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00145, "step": 419, "tokens/total": 13632640, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 190445} +{"epoch": 1.640625, "grad_norm": 0.005391040816903114, "learning_rate": 1.785823392239424e-05, "loss": 6.259237125050277e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00006, "step": 420, "tokens/total": 13664920, "tokens/train_per_sec_per_gpu": 10.06, "tokens/trainable": 190886} +{"epoch": 1.64453125, "grad_norm": 0.13418199121952057, "learning_rate": 1.7695112982104225e-05, "loss": 0.001897090463899076, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.0019, "step": 421, "tokens/total": 13697144, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 191325} +{"epoch": 1.6484375, "grad_norm": 0.0016849171370267868, "learning_rate": 1.7533544450435433e-05, "loss": 4.5368506107479334e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00005, "step": 422, "tokens/total": 13724496, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 191758} +{"epoch": 1.65234375, "grad_norm": 0.0015177769819274545, "learning_rate": 1.7373535050913946e-05, "loss": 3.603554068831727e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00004, "step": 423, "tokens/total": 13757880, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 192195} +{"epoch": 1.65625, "grad_norm": 0.02441447600722313, "learning_rate": 1.721509144218405e-05, "loss": 0.00028592708986252546, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00029, "step": 424, "tokens/total": 13787648, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 192625} +{"epoch": 1.66015625, "grad_norm": 0.00818200595676899, "learning_rate": 1.705822021773101e-05, "loss": 0.00011484129208838567, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00011, "step": 425, "tokens/total": 13819720, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 193111} +{"epoch": 1.6640625, "grad_norm": 0.04355742782354355, "learning_rate": 1.69029279056068e-05, "loss": 0.00018994903075508773, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00019, "step": 426, "tokens/total": 13854944, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 193551} +{"epoch": 1.66796875, "grad_norm": 0.003307729959487915, "learning_rate": 1.6749220968158415e-05, "loss": 6.571791163878515e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00007, "step": 427, "tokens/total": 13887824, "tokens/train_per_sec_per_gpu": 10.43, "tokens/trainable": 194057} +{"epoch": 1.671875, "grad_norm": 0.006264663767069578, "learning_rate": 1.659710580175893e-05, "loss": 8.726881787879393e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00009, "step": 428, "tokens/total": 13918192, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 194556} +{"epoch": 1.67578125, "grad_norm": 0.00803372636437416, "learning_rate": 1.644658873654133e-05, "loss": 0.00014889161684550345, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00015, "step": 429, "tokens/total": 13947440, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 194993} +{"epoch": 1.6796875, "grad_norm": 0.005007718689739704, "learning_rate": 1.629767603613508e-05, "loss": 9.694135951576754e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.0001, "step": 430, "tokens/total": 13974360, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 195393} +{"epoch": 1.68359375, "grad_norm": 0.00255338242277503, "learning_rate": 1.615037389740547e-05, "loss": 5.895699359825812e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00006, "step": 431, "tokens/total": 14006592, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 195837} +{"epoch": 1.6875, "grad_norm": 0.0049398052506148815, "learning_rate": 1.600468845019576e-05, "loss": 8.838514622766525e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00009, "step": 432, "tokens/total": 14040776, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 196317} +{"epoch": 1.69140625, "grad_norm": 0.0006911171367391944, "learning_rate": 1.5860625757072092e-05, "loss": 1.8009348423220217e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00002, "step": 433, "tokens/total": 14073040, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 196765} +{"epoch": 1.6953125, "grad_norm": 0.003309129737317562, "learning_rate": 1.571819181307116e-05, "loss": 6.189122359501198e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00006, "step": 434, "tokens/total": 14107624, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 197237} +{"epoch": 1.69921875, "grad_norm": 0.28145232796669006, "learning_rate": 1.557739254545075e-05, "loss": 0.0009323864942416549, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00093, "step": 435, "tokens/total": 14137784, "tokens/train_per_sec_per_gpu": 10.72, "tokens/trainable": 197709} +{"epoch": 1.703125, "grad_norm": 0.08039931207895279, "learning_rate": 1.543823381344311e-05, "loss": 0.0008910549804568291, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00089, "step": 436, "tokens/total": 14167776, "tokens/train_per_sec_per_gpu": 10.85, "tokens/trainable": 198184} +{"epoch": 1.70703125, "grad_norm": 0.002483709715306759, "learning_rate": 1.5300721408011114e-05, "loss": 5.95565143157728e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00006, "step": 437, "tokens/total": 14199752, "tokens/train_per_sec_per_gpu": 10.31, "tokens/trainable": 198641} +{"epoch": 1.7109375, "grad_norm": 0.0038068110588937998, "learning_rate": 1.5164861051607254e-05, "loss": 7.863929204177111e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.00008, "step": 438, "tokens/total": 14230344, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 199049} +{"epoch": 1.71484375, "grad_norm": 0.030095618218183517, "learning_rate": 1.5030658397935521e-05, "loss": 0.00020675075938925147, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00021, "step": 439, "tokens/total": 14263808, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 199518} +{"epoch": 1.71875, "grad_norm": 0.003213925287127495, "learning_rate": 1.4898119031716104e-05, "loss": 6.764936551917344e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00007, "step": 440, "tokens/total": 14295864, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 199976} +{"epoch": 1.72265625, "grad_norm": 0.015026135370135307, "learning_rate": 1.476724846845306e-05, "loss": 0.00016304825840052217, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00016, "step": 441, "tokens/total": 14331688, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 200453} +{"epoch": 1.7265625, "grad_norm": 0.02206992357969284, "learning_rate": 1.463805215420471e-05, "loss": 0.0002604349865578115, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00026, "step": 442, "tokens/total": 14364688, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 200923} +{"epoch": 1.73046875, "grad_norm": 0.4120462238788605, "learning_rate": 1.451053546535705e-05, "loss": 0.0020386110991239548, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00204, "step": 443, "tokens/total": 14397832, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 201379} +{"epoch": 1.734375, "grad_norm": 0.02033100090920925, "learning_rate": 1.438470370840001e-05, "loss": 0.00018926487246062607, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00019, "step": 444, "tokens/total": 14431864, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 201851} +{"epoch": 1.73828125, "grad_norm": 0.09828748553991318, "learning_rate": 1.4260562119706606e-05, "loss": 0.000972592388279736, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00097, "step": 445, "tokens/total": 14462152, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 202325} +{"epoch": 1.7421875, "grad_norm": 0.013372463174164295, "learning_rate": 1.413811586531508e-05, "loss": 0.00015476249973289669, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00015, "step": 446, "tokens/total": 14493072, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 202766} +{"epoch": 1.74609375, "grad_norm": 0.010553518310189247, "learning_rate": 1.4017370040713884e-05, "loss": 0.00011337868636474013, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00011, "step": 447, "tokens/total": 14527392, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 203215} +{"epoch": 1.75, "grad_norm": 0.015193897299468517, "learning_rate": 1.3898329670629645e-05, "loss": 0.00016131297161336988, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00016, "step": 448, "tokens/total": 14561992, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 203660} +{"epoch": 1.75390625, "grad_norm": 0.0031925903167575598, "learning_rate": 1.3780999708818058e-05, "loss": 5.889539170311764e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00006, "step": 449, "tokens/total": 14592056, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 204110} +{"epoch": 1.7578125, "grad_norm": 0.004992574453353882, "learning_rate": 1.3665385037857758e-05, "loss": 7.14480847818777e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00007, "step": 450, "tokens/total": 14622816, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 204562} +{"epoch": 1.76171875, "grad_norm": 0.12705092132091522, "learning_rate": 1.3551490468947126e-05, "loss": 0.0008850583690218627, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00089, "step": 451, "tokens/total": 14653272, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 205008} +{"epoch": 1.765625, "grad_norm": 0.022879209369421005, "learning_rate": 1.3439320741704075e-05, "loss": 0.000266610732069239, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00027, "step": 452, "tokens/total": 14683952, "tokens/train_per_sec_per_gpu": 10.11, "tokens/trainable": 205432} +{"epoch": 1.76953125, "grad_norm": 0.012072284705936909, "learning_rate": 1.3328880523968808e-05, "loss": 0.00016627777949906886, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00017, "step": 453, "tokens/total": 14716256, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 205895} +{"epoch": 1.7734375, "grad_norm": 0.2610284686088562, "learning_rate": 1.3220174411609587e-05, "loss": 0.005537560675293207, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00555, "step": 454, "tokens/total": 14749088, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 206334} +{"epoch": 1.77734375, "grad_norm": 0.02541235461831093, "learning_rate": 1.3113206928331471e-05, "loss": 0.00021532000391744077, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00022, "step": 455, "tokens/total": 14777384, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 206796} +{"epoch": 1.78125, "grad_norm": 0.013894759118556976, "learning_rate": 1.300798252548806e-05, "loss": 3.743942943401635e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00004, "step": 456, "tokens/total": 14809640, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 207248} +{"epoch": 1.78515625, "grad_norm": 0.19966518878936768, "learning_rate": 1.2904505581896265e-05, "loss": 0.014283248223364353, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01439, "step": 457, "tokens/total": 14843728, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 207721} +{"epoch": 1.7890625, "grad_norm": 0.015443709678947926, "learning_rate": 1.2802780403654082e-05, "loss": 0.00027237783069722354, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00027, "step": 458, "tokens/total": 14876256, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 208168} +{"epoch": 1.79296875, "grad_norm": 0.015815963968634605, "learning_rate": 1.2702811223961408e-05, "loss": 0.00012163716019131243, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00012, "step": 459, "tokens/total": 14907000, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 208629} +{"epoch": 1.796875, "grad_norm": 0.0025485216174274683, "learning_rate": 1.2604602202943861e-05, "loss": 5.099789996165782e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00005, "step": 460, "tokens/total": 14937704, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 209073} +{"epoch": 1.80078125, "grad_norm": 0.012944391928613186, "learning_rate": 1.2508157427479686e-05, "loss": 7.981026283232495e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00008, "step": 461, "tokens/total": 14966920, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 209520} +{"epoch": 1.8046875, "grad_norm": 0.004954006988555193, "learning_rate": 1.2413480911029655e-05, "loss": 4.6060078602749854e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00005, "step": 462, "tokens/total": 14996040, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 209958} +{"epoch": 1.80859375, "grad_norm": 0.256049245595932, "learning_rate": 1.2320576593470082e-05, "loss": 0.016339832916855812, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.01647, "step": 463, "tokens/total": 15029096, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 210385} +{"epoch": 1.8125, "grad_norm": 0.3461129367351532, "learning_rate": 1.2229448340928828e-05, "loss": 0.0034480909816920757, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00345, "step": 464, "tokens/total": 15063696, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 210893} +{"epoch": 1.81640625, "grad_norm": 0.005597686395049095, "learning_rate": 1.2140099945624458e-05, "loss": 8.60400396049954e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00009, "step": 465, "tokens/total": 15098264, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 211350} +{"epoch": 1.8203125, "grad_norm": 0.1975577026605606, "learning_rate": 1.205253512570841e-05, "loss": 0.0015651269350200891, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00157, "step": 466, "tokens/total": 15130464, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 211798} +{"epoch": 1.82421875, "grad_norm": 0.06414192914962769, "learning_rate": 1.1966757525110255e-05, "loss": 0.0006189111736603081, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00062, "step": 467, "tokens/total": 15158736, "tokens/train_per_sec_per_gpu": 10.16, "tokens/trainable": 212249} +{"epoch": 1.828125, "grad_norm": 0.14903585612773895, "learning_rate": 1.1882770713386095e-05, "loss": 0.0014443553518503904, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00145, "step": 468, "tokens/total": 15193648, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 212724} +{"epoch": 1.83203125, "grad_norm": 0.07339369505643845, "learning_rate": 1.180057818556998e-05, "loss": 0.0003746807051356882, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00037, "step": 469, "tokens/total": 15227240, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 213199} +{"epoch": 1.8359375, "grad_norm": 0.08972260355949402, "learning_rate": 1.1720183362028494e-05, "loss": 3.3143987820949405e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00003, "step": 470, "tokens/total": 15259968, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 213664} +{"epoch": 1.83984375, "grad_norm": 1.1099252700805664, "learning_rate": 1.1641589588318387e-05, "loss": 0.001209322945214808, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00121, "step": 471, "tokens/total": 15292480, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 214109} +{"epoch": 1.84375, "grad_norm": 0.0052484916523098946, "learning_rate": 1.1564800135047418e-05, "loss": 4.590555909089744e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00005, "step": 472, "tokens/total": 15324976, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 214597} +{"epoch": 1.84765625, "grad_norm": 0.005445324815809727, "learning_rate": 1.148981819773816e-05, "loss": 5.5425196478608996e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00006, "step": 473, "tokens/total": 15356448, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 215014} +{"epoch": 1.8515625, "grad_norm": 0.01340578030794859, "learning_rate": 1.1416646896695086e-05, "loss": 0.00014287135854829103, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00014, "step": 474, "tokens/total": 15389192, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 215466} +{"epoch": 1.85546875, "grad_norm": 0.00947215873748064, "learning_rate": 1.1345289276874717e-05, "loss": 0.00018046522745862603, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00018, "step": 475, "tokens/total": 15422728, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 215937} +{"epoch": 1.859375, "grad_norm": 0.01584790088236332, "learning_rate": 1.1275748307758873e-05, "loss": 7.881203782744706e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00008, "step": 476, "tokens/total": 15457104, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 216405} +{"epoch": 1.86328125, "grad_norm": 0.09769656509160995, "learning_rate": 1.1208026883231147e-05, "loss": 0.0009415019303560257, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00094, "step": 477, "tokens/total": 15486592, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 216820} +{"epoch": 1.8671875, "grad_norm": 0.011226327158510685, "learning_rate": 1.1142127821456433e-05, "loss": 6.251124432310462e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00006, "step": 478, "tokens/total": 15518368, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 217270} +{"epoch": 1.87109375, "grad_norm": 0.13403664529323578, "learning_rate": 1.1078053864763674e-05, "loss": 0.0009262033272534609, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00093, "step": 479, "tokens/total": 15551256, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 217722} +{"epoch": 1.875, "grad_norm": 0.03326552361249924, "learning_rate": 1.1015807679531756e-05, "loss": 0.00013762044545728713, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00014, "step": 480, "tokens/total": 15581480, "tokens/train_per_sec_per_gpu": 10.04, "tokens/trainable": 218170} +{"epoch": 1.87890625, "grad_norm": 0.27987849712371826, "learning_rate": 1.0955391856078528e-05, "loss": 0.0024562692269682884, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00246, "step": 481, "tokens/total": 15615104, "tokens/train_per_sec_per_gpu": 10.4, "tokens/trainable": 218644} +{"epoch": 1.8828125, "grad_norm": 0.0504269078373909, "learning_rate": 1.0896808908553007e-05, "loss": 0.00034226931165903807, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00034, "step": 482, "tokens/total": 15645384, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 219092} +{"epoch": 1.88671875, "grad_norm": 0.025805030018091202, "learning_rate": 1.0840061274830763e-05, "loss": 0.0002996327239088714, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.0003, "step": 483, "tokens/total": 15676600, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 219558} +{"epoch": 1.890625, "grad_norm": 0.011537019163370132, "learning_rate": 1.0785151316412473e-05, "loss": 4.555026680463925e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00005, "step": 484, "tokens/total": 15709712, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 220007} +{"epoch": 1.89453125, "grad_norm": 0.13657735288143158, "learning_rate": 1.0732081318325639e-05, "loss": 0.0004512836749199778, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00045, "step": 485, "tokens/total": 15747848, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 220505} +{"epoch": 1.8984375, "grad_norm": 0.017776502296328545, "learning_rate": 1.0680853489029501e-05, "loss": 0.0001847639650804922, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00018, "step": 486, "tokens/total": 15782416, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 220936} +{"epoch": 1.90234375, "grad_norm": 0.07781105488538742, "learning_rate": 1.0631469960323152e-05, "loss": 0.0004961894010193646, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.0005, "step": 487, "tokens/total": 15815456, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 221372} +{"epoch": 1.90625, "grad_norm": 0.016368068754673004, "learning_rate": 1.0583932787256783e-05, "loss": 0.00011522167187649757, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00012, "step": 488, "tokens/total": 15846496, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 221792} +{"epoch": 1.91015625, "grad_norm": 0.00755052175372839, "learning_rate": 1.0538243948046206e-05, "loss": 9.31734248297289e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00009, "step": 489, "tokens/total": 15878784, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 222238} +{"epoch": 1.9140625, "grad_norm": 0.006571550853550434, "learning_rate": 1.0494405343990523e-05, "loss": 4.062007792526856e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00004, "step": 490, "tokens/total": 15909616, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 222682} +{"epoch": 1.91796875, "grad_norm": 0.00406951317563653, "learning_rate": 1.0452418799392985e-05, "loss": 6.480566662503406e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00006, "step": 491, "tokens/total": 15941400, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 223142} +{"epoch": 1.921875, "grad_norm": 0.21290186047554016, "learning_rate": 1.0412286061485102e-05, "loss": 0.003370263846591115, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00338, "step": 492, "tokens/total": 15975440, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 223609} +{"epoch": 1.92578125, "grad_norm": 0.0024910836946219206, "learning_rate": 1.03740088003539e-05, "loss": 3.833818482235074e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00004, "step": 493, "tokens/total": 16004224, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 224050} +{"epoch": 1.9296875, "grad_norm": 0.0024206729140132666, "learning_rate": 1.0337588608872463e-05, "loss": 4.641913255909458e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00005, "step": 494, "tokens/total": 16032976, "tokens/train_per_sec_per_gpu": 10.6, "tokens/trainable": 224509} +{"epoch": 1.93359375, "grad_norm": 0.38814637064933777, "learning_rate": 1.0303027002633622e-05, "loss": 0.009800954721868038, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00985, "step": 495, "tokens/total": 16065312, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 225007} +{"epoch": 1.9375, "grad_norm": 0.0034924878273159266, "learning_rate": 1.0270325419886884e-05, "loss": 3.967645898228511e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00004, "step": 496, "tokens/total": 16097552, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 225461} +{"epoch": 1.94140625, "grad_norm": 0.2767745554447174, "learning_rate": 1.0239485221478599e-05, "loss": 0.002217609202489257, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00222, "step": 497, "tokens/total": 16132112, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 225924} +{"epoch": 1.9453125, "grad_norm": 0.1384720355272293, "learning_rate": 1.0210507690795292e-05, "loss": 0.0007754692924208939, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00078, "step": 498, "tokens/total": 16163632, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 226349} +{"epoch": 1.94921875, "grad_norm": 0.10108991712331772, "learning_rate": 1.0183394033710305e-05, "loss": 0.0010362070752307773, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00104, "step": 499, "tokens/total": 16194632, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 226800} +{"epoch": 1.953125, "grad_norm": 0.023618271574378014, "learning_rate": 1.0158145378533583e-05, "loss": 0.00022886654187459499, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00023, "step": 500, "tokens/total": 16225280, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 227267} +{"epoch": 1.95703125, "grad_norm": 0.006232099141925573, "learning_rate": 1.0134762775964726e-05, "loss": 4.978999641025439e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00005, "step": 501, "tokens/total": 16257400, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 227757} +{"epoch": 1.9609375, "grad_norm": 0.030084781348705292, "learning_rate": 1.0113247199049278e-05, "loss": 0.00022654666099697351, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00023, "step": 502, "tokens/total": 16290472, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 228238} +{"epoch": 1.96484375, "grad_norm": 0.006856183055788279, "learning_rate": 1.0093599543138205e-05, "loss": 9.10824237507768e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00009, "step": 503, "tokens/total": 16324232, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 228715} +{"epoch": 1.96875, "grad_norm": 0.005263664294034243, "learning_rate": 1.0075820625850675e-05, "loss": 9.17431534617208e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00009, "step": 504, "tokens/total": 16360464, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 229165} +{"epoch": 1.97265625, "grad_norm": 0.004934739787131548, "learning_rate": 1.0059911187040013e-05, "loss": 7.883762009441853e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00008, "step": 505, "tokens/total": 16393136, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 229597} +{"epoch": 1.9765625, "grad_norm": 0.034243177622556686, "learning_rate": 1.0045871888762893e-05, "loss": 0.00018137787992600352, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00018, "step": 506, "tokens/total": 16428488, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 230066} +{"epoch": 1.98046875, "grad_norm": 0.024661850184202194, "learning_rate": 1.003370331525184e-05, "loss": 0.00030833540949970484, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00031, "step": 507, "tokens/total": 16462616, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 230537} +{"epoch": 1.984375, "grad_norm": 0.0051201279275119305, "learning_rate": 1.002340597289085e-05, "loss": 1.5460140275536105e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00002, "step": 508, "tokens/total": 16495600, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 230987} +{"epoch": 1.98828125, "grad_norm": 0.0075493729673326015, "learning_rate": 1.0014980290194387e-05, "loss": 8.388420974370092e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00008, "step": 509, "tokens/total": 16525504, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 231427} +{"epoch": 1.9921875, "grad_norm": 0.0070311930030584335, "learning_rate": 1.0008426617789489e-05, "loss": 8.010146848391742e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00008, "step": 510, "tokens/total": 16556960, "tokens/train_per_sec_per_gpu": 10.2, "tokens/trainable": 231889} +{"epoch": 1.99609375, "grad_norm": 0.027394186705350876, "learning_rate": 1.0003745228401215e-05, "loss": 0.00015287323913071305, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00015, "step": 511, "tokens/total": 16588024, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 232351} +{"epoch": 2.0, "grad_norm": 0.0038574975915253162, "learning_rate": 1.0000936316841296e-05, "loss": 5.598958523478359e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00006, "step": 512, "tokens/total": 16622040, "tokens/train_per_sec_per_gpu": 6.53, "tokens/trainable": 232756} diff --git a/gemma3_27b_19m/control/training/aft/mixed_coin/axolotl.yaml b/gemma3_27b_19m/control/training/aft/mixed_coin/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89ceafe4c722b8045781592971bf93651fd06576 --- /dev/null +++ b/gemma3_27b_19m/control/training/aft/mixed_coin/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_27b_19m/control/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_19m/control/aft/mixed_coin/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_19m/control/aft/mixed_coin/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_19m/control/training/aft/mixed_coin/train.log b/gemma3_27b_19m/control/training/aft/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..759cf40b8d94153f3227b0083a7497c1905bd2b0 --- /dev/null +++ b/gemma3_27b_19m/control/training/aft/mixed_coin/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-03 09:04:30,443] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 09:04:33.156000 273174 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:04:33.177000 273174 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-03 09:04:42,627] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 09:04:45.768000 274224 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:04:45.787000 274224 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-03 09:04:48,191] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-03 09:04:48,193] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-03 09:04:48,193] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 09:04:48,212] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 09:04:48,254] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-03 09:04:48,255] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-03 09:04:48,405] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_19m/control/aft/mixed_coin/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_19m/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/control/aft/mixed_coin/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/control/aft/mixed_coin/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-03 09:04:50,379] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_19m/control/aft/mixed_coin/prepared/b51d082027204c7dee2948d5a6c84740 +[2026-09-03 09:04:50,379] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-03 09:04:50,379] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 39276.39 examples/s] Generating train split: 8002 examples [00:00, 37816.09 examples/s] Generating train split: 8192 examples [00:00, 37212.16 examples/s] +[2026-09-03 09:04:51,134] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-03 09:04:51,136] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 2977.11 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 15431.05 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:59.183000 274922 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:59.554000 274917 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:59.573000 274917 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:59.821000 274913 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:59.840000 274913 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:59.892000 274916 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:05:59.911000 274916 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:00.056000 274914 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:00.074000 274914 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:00.379000 274912 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:00.397000 274912 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:01.456000 274918 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:01.475000 274918 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.201000 274911 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.219000 274911 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.544000 274915 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 09:06:02.564000 274915 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:16<01:58, 60.61 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:16<01:58, 60.61 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:16<01:41, 60.61 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:16<01:24, 60.61 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:16<01:07, 60.61 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:16<00:50, 60.61 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:16<00:33, 60.61 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:16<00:16, 60.61 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:16<00:00, 60.61 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:18<00:00, 452.38 examples/s] +[2026-09-03 09:06:06,180] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-03 09:06:13,757] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/control/aft/mixed_coin/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/control/aft/mixed_coin/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_19m/control/aft/mixed_coin/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "exists": true, + "size_bytes": 21469589, + "sha256": "e42045fcedcb63fae01be1c0b8fc72ec9b77575169384a7f7deae36a58c5bf53", + "nonempty_rows": 8192, + "ordered_example_sha256": "e88d1f8b9a1f16a74a384a55cd25883cef3ba90cdc5636fed6746152ac88d691", + "example_manifest": "/workspace/final_v1/gemma3_27b_19m/control/aft/mixed_coin/training_examples.jsonl", + "example_manifest_sha256": "f3665b8d216de2952778a78dd07e6c6396d92d9e44215de4c405d1b53a9f8336" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T10:58:36+00:00", + "resolved_config_sha256": "e66cd114e76c5f62e6ff28a6544bcb9b2a9cba9ecb95e645d42deeb620b5e970", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.6180773049989663e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_19m/control/aft/mixed_coin/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_19m/control/aft/mixed_coin/trainer_state.final.json", + "trainer_state_sha256": "8efb276a9f3f299b32081084f11b8689c6cfc7f0a82d5cfac2214d0a11903229", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_19m/control/aft/mixed_coin/training_trace.jsonl", + "trace_sha256": "3b9a7d38536a5972f520e4b60ea97b291ec68803845b7e6c9deebc3cb06dba70", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_19m/control/training/aft/mixed_coin/training_trace.jsonl b/gemma3_27b_19m/control/training/aft/mixed_coin/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..84984fbc2461b577ea9c029f17544b3fe5261125 --- /dev/null +++ b/gemma3_27b_19m/control/training/aft/mixed_coin/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 2.4540858268737793, "learning_rate": 0.0, "loss": 0.22921109199523926, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.25761, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 5.987685680389404, "learning_rate": 4.000000000000001e-06, "loss": 0.2750456631183624, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.31659, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 2.644988536834717, "learning_rate": 8.000000000000001e-06, "loss": 0.25217416882514954, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.28682, "step": 3, "tokens/total": 92448, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 1380} +{"epoch": 0.015625, "grad_norm": 2.527042865753174, "learning_rate": 1.2e-05, "loss": 0.23517084121704102, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.26512, "step": 4, "tokens/total": 122888, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 1834} +{"epoch": 0.01953125, "grad_norm": 2.728114128112793, "learning_rate": 1.6000000000000003e-05, "loss": 0.2400764524936676, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.27135, "step": 5, "tokens/total": 158184, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 2286} +{"epoch": 0.0234375, "grad_norm": 1.7906097173690796, "learning_rate": 2e-05, "loss": 0.19823648035526276, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.21925, "step": 6, "tokens/total": 188096, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 2726} +{"epoch": 0.02734375, "grad_norm": 3.33052659034729, "learning_rate": 2.4e-05, "loss": 0.16783061623573303, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.18274, "step": 7, "tokens/total": 220488, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.540222406387329, "learning_rate": 2.8000000000000003e-05, "loss": 0.13140103220939636, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.14043, "step": 8, "tokens/total": 251488, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 2.0588390827178955, "learning_rate": 3.2000000000000005e-05, "loss": 0.10534863919019699, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.1111, "step": 9, "tokens/total": 279512, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 1.69157874584198, "learning_rate": 3.6e-05, "loss": 0.1164781004190445, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.12353, "step": 10, "tokens/total": 310824, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 3.508064031600952, "learning_rate": 4e-05, "loss": 0.08756688237190247, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.09152, "step": 11, "tokens/total": 343872, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 3.7288873195648193, "learning_rate": 4.4000000000000006e-05, "loss": 0.12498999387025833, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.13314, "step": 12, "tokens/total": 378144, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 5452} +{"epoch": 0.05078125, "grad_norm": 4.1545844078063965, "learning_rate": 4.8e-05, "loss": 0.06380931288003922, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.06589, "step": 13, "tokens/total": 406912, "tokens/train_per_sec_per_gpu": 10.15, "tokens/trainable": 5891} +{"epoch": 0.0546875, "grad_norm": 2.1959195137023926, "learning_rate": 5.2000000000000004e-05, "loss": 0.059787943959236145, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.06161, "step": 14, "tokens/total": 437952, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 6327} +{"epoch": 0.05859375, "grad_norm": 2.292597532272339, "learning_rate": 5.6000000000000006e-05, "loss": 0.047135282307863235, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.04826, "step": 15, "tokens/total": 471248, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 1.6937998533248901, "learning_rate": 6e-05, "loss": 0.055725548416376114, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.05731, "step": 16, "tokens/total": 502120, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 4.420352458953857, "learning_rate": 6.400000000000001e-05, "loss": 0.04702301323413849, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.04815, "step": 17, "tokens/total": 535976, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 2.8663887977600098, "learning_rate": 6.800000000000001e-05, "loss": 0.04992178827524185, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.05119, "step": 18, "tokens/total": 570400, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 8124} +{"epoch": 0.07421875, "grad_norm": 1.3499531745910645, "learning_rate": 7.2e-05, "loss": 0.031148400157690048, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.03164, "step": 19, "tokens/total": 603640, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 8567} +{"epoch": 0.078125, "grad_norm": 0.9975746870040894, "learning_rate": 7.6e-05, "loss": 0.03388882428407669, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.03447, "step": 20, "tokens/total": 636520, "tokens/train_per_sec_per_gpu": 6.65, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.8749074339866638, "learning_rate": 8e-05, "loss": 0.018754614517092705, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.01893, "step": 21, "tokens/total": 667016, "tokens/train_per_sec_per_gpu": 10.76, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 1.2482085227966309, "learning_rate": 8.4e-05, "loss": 0.019186416640877724, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.01937, "step": 22, "tokens/total": 698840, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 0.8607750535011292, "learning_rate": 8.800000000000001e-05, "loss": 0.024713683873414993, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.02502, "step": 23, "tokens/total": 731616, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 1.5459352731704712, "learning_rate": 9.200000000000001e-05, "loss": 0.023936238139867783, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.02423, "step": 24, "tokens/total": 761952, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 3.170828104019165, "learning_rate": 9.6e-05, "loss": 0.088631272315979, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.09268, "step": 25, "tokens/total": 792240, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 6.860727310180664, "learning_rate": 0.0001, "loss": 0.03809942677617073, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.03883, "step": 26, "tokens/total": 819696, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 1.4224698543548584, "learning_rate": 9.99990636831587e-05, "loss": 0.047907132655382156, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.04907, "step": 27, "tokens/total": 851176, "tokens/train_per_sec_per_gpu": 10.35, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 4.666821479797363, "learning_rate": 9.999625477159879e-05, "loss": 0.027108553797006607, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.02748, "step": 28, "tokens/total": 886496, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 0.7940762042999268, "learning_rate": 9.999157338221051e-05, "loss": 0.015243424102663994, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.01536, "step": 29, "tokens/total": 918416, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 3.428521156311035, "learning_rate": 9.998501970980562e-05, "loss": 0.025058068335056305, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.02537, "step": 30, "tokens/total": 948512, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 4.42432165145874, "learning_rate": 9.997659402710915e-05, "loss": 0.011492185294628143, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.01156, "step": 31, "tokens/total": 982400, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 1.200294017791748, "learning_rate": 9.996629668474818e-05, "loss": 0.029902726411819458, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.03035, "step": 32, "tokens/total": 1016168, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 4.618226051330566, "learning_rate": 9.995412811123711e-05, "loss": 0.020390082150697708, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.0206, "step": 33, "tokens/total": 1048376, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 1.1908762454986572, "learning_rate": 9.994008881295999e-05, "loss": 0.024883093312382698, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.0252, "step": 34, "tokens/total": 1080792, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 1.0725144147872925, "learning_rate": 9.992417937414932e-05, "loss": 0.028854668140411377, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.02927, "step": 35, "tokens/total": 1112144, "tokens/train_per_sec_per_gpu": 10.38, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 1.0840307474136353, "learning_rate": 9.99064004568618e-05, "loss": 0.025260351598262787, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.02558, "step": 36, "tokens/total": 1144776, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 0.463015079498291, "learning_rate": 9.988675280095074e-05, "loss": 0.011643484234809875, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.01171, "step": 37, "tokens/total": 1175576, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.511925995349884, "learning_rate": 9.986523722403528e-05, "loss": 0.017768777906894684, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01793, "step": 38, "tokens/total": 1206008, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.6454110741615295, "learning_rate": 9.984185462146642e-05, "loss": 0.02985622175037861, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.03031, "step": 39, "tokens/total": 1238280, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 0.7829225063323975, "learning_rate": 9.98166059662897e-05, "loss": 0.020356398075819016, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.02057, "step": 40, "tokens/total": 1267680, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 0.23417702317237854, "learning_rate": 9.978949230920472e-05, "loss": 0.007772943004965782, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.0078, "step": 41, "tokens/total": 1300168, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 1.8820881843566895, "learning_rate": 9.976051477852141e-05, "loss": 0.04591422528028488, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.04698, "step": 42, "tokens/total": 1332960, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 0.8636189699172974, "learning_rate": 9.972967458011312e-05, "loss": 0.0273672454059124, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.02775, "step": 43, "tokens/total": 1365160, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 0.9235569834709167, "learning_rate": 9.96969729973664e-05, "loss": 0.02431337535381317, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.02461, "step": 44, "tokens/total": 1402920, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 20001} +{"epoch": 0.17578125, "grad_norm": 0.46154043078422546, "learning_rate": 9.966241139112754e-05, "loss": 0.014735816046595573, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.01484, "step": 45, "tokens/total": 1434976, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 20442} +{"epoch": 0.1796875, "grad_norm": 0.3423166573047638, "learning_rate": 9.96259911996461e-05, "loss": 0.008961593732237816, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.009, "step": 46, "tokens/total": 1466704, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 20874} +{"epoch": 0.18359375, "grad_norm": 2.011575222015381, "learning_rate": 9.958771393851491e-05, "loss": 0.06303266435861588, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.06506, "step": 47, "tokens/total": 1497856, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 21315} +{"epoch": 0.1875, "grad_norm": 0.6666603088378906, "learning_rate": 9.954758120060702e-05, "loss": 0.029031958431005478, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.02946, "step": 48, "tokens/total": 1529880, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 21775} +{"epoch": 0.19140625, "grad_norm": 0.4289197623729706, "learning_rate": 9.950559465600948e-05, "loss": 0.007718895096331835, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00775, "step": 49, "tokens/total": 1562128, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 22215} +{"epoch": 0.1953125, "grad_norm": 0.6853031516075134, "learning_rate": 9.946175605195379e-05, "loss": 0.01606380008161068, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01619, "step": 50, "tokens/total": 1597344, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 22700} +{"epoch": 0.19921875, "grad_norm": 0.5193058252334595, "learning_rate": 9.941606721274322e-05, "loss": 0.013592232018709183, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.01369, "step": 51, "tokens/total": 1629896, "tokens/train_per_sec_per_gpu": 6.97, "tokens/trainable": 23135} +{"epoch": 0.203125, "grad_norm": 1.078377604484558, "learning_rate": 9.936853003967685e-05, "loss": 0.022813137620687485, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.02308, "step": 52, "tokens/total": 1663816, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 1.429619550704956, "learning_rate": 9.93191465109705e-05, "loss": 0.034881263971328735, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.0355, "step": 53, "tokens/total": 1696744, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 0.4397701323032379, "learning_rate": 9.926791868167438e-05, "loss": 0.012500951066613197, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.01258, "step": 54, "tokens/total": 1731552, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 1.4957741498947144, "learning_rate": 9.921484868358753e-05, "loss": 0.05830182135105133, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.06003, "step": 55, "tokens/total": 1763208, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 1.8276586532592773, "learning_rate": 9.915993872516924e-05, "loss": 0.03218724951148033, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.03271, "step": 56, "tokens/total": 1796120, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.6707075238227844, "learning_rate": 9.9103191091447e-05, "loss": 0.01353541947901249, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01363, "step": 57, "tokens/total": 1826048, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.3623991012573242, "learning_rate": 9.904460814392147e-05, "loss": 0.011652317829430103, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.01172, "step": 58, "tokens/total": 1863152, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.3095373511314392, "learning_rate": 9.898419232046825e-05, "loss": 0.012743961997330189, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01283, "step": 59, "tokens/total": 1898008, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 1.6066689491271973, "learning_rate": 9.892194613523633e-05, "loss": 0.01948057860136032, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01967, "step": 60, "tokens/total": 1932616, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.41761839389801025, "learning_rate": 9.885787217854357e-05, "loss": 0.008680928498506546, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00872, "step": 61, "tokens/total": 1964544, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.5800505876541138, "learning_rate": 9.879197311676887e-05, "loss": 0.027204688638448715, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.02758, "step": 62, "tokens/total": 1997272, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 1.4490835666656494, "learning_rate": 9.872425169224113e-05, "loss": 0.03626297786831856, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.03693, "step": 63, "tokens/total": 2029744, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 28623} +{"epoch": 0.25, "grad_norm": 1.4844439029693604, "learning_rate": 9.865471072312528e-05, "loss": 0.01831192895770073, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.01848, "step": 64, "tokens/total": 2065136, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 29108} +{"epoch": 0.25390625, "grad_norm": 0.9702927470207214, "learning_rate": 9.858335310330492e-05, "loss": 0.008437048643827438, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00847, "step": 65, "tokens/total": 2094864, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 29570} +{"epoch": 0.2578125, "grad_norm": 0.686852216720581, "learning_rate": 9.851018180226185e-05, "loss": 0.016076913103461266, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01621, "step": 66, "tokens/total": 2126864, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 30011} +{"epoch": 0.26171875, "grad_norm": 0.6618979573249817, "learning_rate": 9.843519986495259e-05, "loss": 0.015020047314465046, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.01513, "step": 67, "tokens/total": 2157816, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 30446} +{"epoch": 0.265625, "grad_norm": 1.7101720571517944, "learning_rate": 9.835841041168162e-05, "loss": 0.009825456887483597, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00987, "step": 68, "tokens/total": 2192664, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 30913} +{"epoch": 0.26953125, "grad_norm": 1.1480623483657837, "learning_rate": 9.82798166379715e-05, "loss": 0.009499817155301571, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.00955, "step": 69, "tokens/total": 2223792, "tokens/train_per_sec_per_gpu": 9.88, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.6748724579811096, "learning_rate": 9.819942181443002e-05, "loss": 0.008641454391181469, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.00868, "step": 70, "tokens/total": 2256288, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 31844} +{"epoch": 0.27734375, "grad_norm": 0.38938653469085693, "learning_rate": 9.811722928661392e-05, "loss": 0.007619414012879133, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.00765, "step": 71, "tokens/total": 2284168, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.22191470861434937, "learning_rate": 9.803324247488975e-05, "loss": 0.002264246344566345, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00227, "step": 72, "tokens/total": 2316496, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.5455390214920044, "learning_rate": 9.794746487429161e-05, "loss": 0.0032744253985583782, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.00328, "step": 73, "tokens/total": 2347680, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.6860706210136414, "learning_rate": 9.785990005437554e-05, "loss": 0.022497761994600296, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.02275, "step": 74, "tokens/total": 2380912, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 1.6671127080917358, "learning_rate": 9.777055165907117e-05, "loss": 0.045735083520412445, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.0468, "step": 75, "tokens/total": 2412000, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 1.030586838722229, "learning_rate": 9.767942340652993e-05, "loss": 0.010158790275454521, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01021, "step": 76, "tokens/total": 2441776, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.5070812702178955, "learning_rate": 9.758651908897035e-05, "loss": 0.014094313606619835, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01419, "step": 77, "tokens/total": 2473896, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 34894} +{"epoch": 0.3046875, "grad_norm": 1.4636118412017822, "learning_rate": 9.749184257252033e-05, "loss": 0.05520675703883171, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.05676, "step": 78, "tokens/total": 2505568, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 35334} +{"epoch": 0.30859375, "grad_norm": 0.7801726460456848, "learning_rate": 9.739539779705614e-05, "loss": 0.013782399706542492, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.01388, "step": 79, "tokens/total": 2542120, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 35829} +{"epoch": 0.3125, "grad_norm": 0.21737253665924072, "learning_rate": 9.729718877603861e-05, "loss": 0.00462491437792778, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00464, "step": 80, "tokens/total": 2572712, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 36260} +{"epoch": 0.31640625, "grad_norm": 0.4189910292625427, "learning_rate": 9.719721959634592e-05, "loss": 0.016941599547863007, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.01709, "step": 81, "tokens/total": 2604304, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 36697} +{"epoch": 0.3203125, "grad_norm": 0.3195907771587372, "learning_rate": 9.709549441810375e-05, "loss": 0.008006863296031952, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00804, "step": 82, "tokens/total": 2639072, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 37181} +{"epoch": 0.32421875, "grad_norm": 0.5605000853538513, "learning_rate": 9.699201747451195e-05, "loss": 0.023583712056279182, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.02386, "step": 83, "tokens/total": 2670240, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 37629} +{"epoch": 0.328125, "grad_norm": 0.4706173539161682, "learning_rate": 9.688679307166854e-05, "loss": 0.006801845505833626, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00683, "step": 84, "tokens/total": 2701112, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 38092} +{"epoch": 0.33203125, "grad_norm": 0.706518292427063, "learning_rate": 9.677982558839042e-05, "loss": 0.02718331106007099, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.02756, "step": 85, "tokens/total": 2733488, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 38571} +{"epoch": 0.3359375, "grad_norm": 0.6121541857719421, "learning_rate": 9.66711194760312e-05, "loss": 0.011174722574651241, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01124, "step": 86, "tokens/total": 2769880, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 39059} +{"epoch": 0.33984375, "grad_norm": 0.46146541833877563, "learning_rate": 9.656067925829593e-05, "loss": 0.01945994235575199, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.01965, "step": 87, "tokens/total": 2799384, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 39519} +{"epoch": 0.34375, "grad_norm": 1.9660956859588623, "learning_rate": 9.644850953105288e-05, "loss": 0.030134903267025948, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.03059, "step": 88, "tokens/total": 2828608, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 39961} +{"epoch": 0.34765625, "grad_norm": 0.40043914318084717, "learning_rate": 9.633461496214225e-05, "loss": 0.005719275679439306, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00574, "step": 89, "tokens/total": 2860464, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 40431} +{"epoch": 0.3515625, "grad_norm": 0.34779444336891174, "learning_rate": 9.621900029118195e-05, "loss": 0.021225523203611374, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.02145, "step": 90, "tokens/total": 2889528, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 40854} +{"epoch": 0.35546875, "grad_norm": 2.29736328125, "learning_rate": 9.610167032937036e-05, "loss": 0.009165533818304539, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00921, "step": 91, "tokens/total": 2923880, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 41373} +{"epoch": 0.359375, "grad_norm": 0.6588590145111084, "learning_rate": 9.598262995928611e-05, "loss": 0.008133485913276672, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00817, "step": 92, "tokens/total": 2953944, "tokens/train_per_sec_per_gpu": 10.05, "tokens/trainable": 41842} +{"epoch": 0.36328125, "grad_norm": 0.3498712182044983, "learning_rate": 9.586188413468492e-05, "loss": 0.006708888802677393, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00673, "step": 93, "tokens/total": 2988120, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 42288} +{"epoch": 0.3671875, "grad_norm": 0.4600358307361603, "learning_rate": 9.57394378802934e-05, "loss": 0.008056784979999065, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00809, "step": 94, "tokens/total": 3020416, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 42740} +{"epoch": 0.37109375, "grad_norm": 0.4623071253299713, "learning_rate": 9.56152962916e-05, "loss": 0.01572243869304657, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.01585, "step": 95, "tokens/total": 3048800, "tokens/train_per_sec_per_gpu": 10.08, "tokens/trainable": 43138} +{"epoch": 0.375, "grad_norm": 0.17185652256011963, "learning_rate": 9.548946453464296e-05, "loss": 0.0035926373675465584, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.0036, "step": 96, "tokens/total": 3081768, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 43623} +{"epoch": 0.37890625, "grad_norm": 0.9573922157287598, "learning_rate": 9.53619478457953e-05, "loss": 0.025939898565411568, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.02628, "step": 97, "tokens/total": 3115608, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 44079} +{"epoch": 0.3828125, "grad_norm": 0.6713507771492004, "learning_rate": 9.523275153154695e-05, "loss": 0.013511577621102333, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.0136, "step": 98, "tokens/total": 3146880, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 44529} +{"epoch": 0.38671875, "grad_norm": 0.7951626777648926, "learning_rate": 9.51018809682839e-05, "loss": 0.02856714278459549, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.02898, "step": 99, "tokens/total": 3180088, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 44976} +{"epoch": 0.390625, "grad_norm": 0.414963036775589, "learning_rate": 9.49693416020645e-05, "loss": 0.007551524788141251, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00758, "step": 100, "tokens/total": 3213968, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 45429} +{"epoch": 0.39453125, "grad_norm": 0.24136435985565186, "learning_rate": 9.483513894839276e-05, "loss": 0.00556909991428256, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00558, "step": 101, "tokens/total": 3245976, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 45876} +{"epoch": 0.3984375, "grad_norm": 0.5384964942932129, "learning_rate": 9.469927859198888e-05, "loss": 0.012666633352637291, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01275, "step": 102, "tokens/total": 3278792, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 46363} +{"epoch": 0.40234375, "grad_norm": 0.36057883501052856, "learning_rate": 9.456176618655689e-05, "loss": 0.011281568557024002, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.01135, "step": 103, "tokens/total": 3309360, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 46810} +{"epoch": 0.40625, "grad_norm": 0.25183814764022827, "learning_rate": 9.442260745454927e-05, "loss": 0.003935391083359718, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00394, "step": 104, "tokens/total": 3344344, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 47287} +{"epoch": 0.41015625, "grad_norm": 0.7326257228851318, "learning_rate": 9.428180818692884e-05, "loss": 0.008091638796031475, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00812, "step": 105, "tokens/total": 3378832, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 47788} +{"epoch": 0.4140625, "grad_norm": 0.3954485058784485, "learning_rate": 9.413937424292791e-05, "loss": 0.010893605649471283, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.01095, "step": 106, "tokens/total": 3415416, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 48243} +{"epoch": 0.41796875, "grad_norm": 0.5276399850845337, "learning_rate": 9.399531154980424e-05, "loss": 0.02094436064362526, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.02117, "step": 107, "tokens/total": 3448120, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 48692} +{"epoch": 0.421875, "grad_norm": 0.31563660502433777, "learning_rate": 9.384962610259455e-05, "loss": 0.016930270940065384, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01707, "step": 108, "tokens/total": 3480800, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.5902420878410339, "learning_rate": 9.370232396386494e-05, "loss": 0.007273034658282995, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.0073, "step": 109, "tokens/total": 3509992, "tokens/train_per_sec_per_gpu": 10.4, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.34283798933029175, "learning_rate": 9.355341126345868e-05, "loss": 0.01146574504673481, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01153, "step": 110, "tokens/total": 3542872, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.5926234126091003, "learning_rate": 9.340289419824107e-05, "loss": 0.01414925791323185, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.01425, "step": 111, "tokens/total": 3573896, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.6282955408096313, "learning_rate": 9.325077903184159e-05, "loss": 0.028679966926574707, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.0291, "step": 112, "tokens/total": 3605648, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.15701688826084137, "learning_rate": 9.30970720943932e-05, "loss": 0.0030540444422513247, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00306, "step": 113, "tokens/total": 3638544, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 51389} +{"epoch": 0.4453125, "grad_norm": 0.4709453284740448, "learning_rate": 9.2941779782269e-05, "loss": 0.0034688706509768963, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00347, "step": 114, "tokens/total": 3670776, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 51862} +{"epoch": 0.44921875, "grad_norm": 0.7868107557296753, "learning_rate": 9.278490855781596e-05, "loss": 0.005834677256643772, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.00585, "step": 115, "tokens/total": 3701472, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 52301} +{"epoch": 0.453125, "grad_norm": 0.37552446126937866, "learning_rate": 9.262646494908604e-05, "loss": 0.010733043774962425, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.01079, "step": 116, "tokens/total": 3735872, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 52755} +{"epoch": 0.45703125, "grad_norm": 0.4894680678844452, "learning_rate": 9.246645554956457e-05, "loss": 0.017127754166722298, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01728, "step": 117, "tokens/total": 3770032, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 53224} +{"epoch": 0.4609375, "grad_norm": 0.5512449741363525, "learning_rate": 9.230488701789578e-05, "loss": 0.023206887766718864, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.02348, "step": 118, "tokens/total": 3805264, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 53666} +{"epoch": 0.46484375, "grad_norm": 0.4399835765361786, "learning_rate": 9.214176607760577e-05, "loss": 0.004129660315811634, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00414, "step": 119, "tokens/total": 3840776, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 54129} +{"epoch": 0.46875, "grad_norm": 0.6103647947311401, "learning_rate": 9.197709951682268e-05, "loss": 0.014887339435517788, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.015, "step": 120, "tokens/total": 3871376, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 54587} +{"epoch": 0.47265625, "grad_norm": 0.8780786395072937, "learning_rate": 9.181089418799428e-05, "loss": 0.03077123686671257, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.03125, "step": 121, "tokens/total": 3904552, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 55070} +{"epoch": 0.4765625, "grad_norm": 0.3713451027870178, "learning_rate": 9.164315700760271e-05, "loss": 0.011941089294850826, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01201, "step": 122, "tokens/total": 3937344, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 55494} +{"epoch": 0.48046875, "grad_norm": 0.3286852240562439, "learning_rate": 9.147389495587671e-05, "loss": 0.015093688853085041, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01521, "step": 123, "tokens/total": 3967392, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 55940} +{"epoch": 0.484375, "grad_norm": 1.1287835836410522, "learning_rate": 9.130311507650116e-05, "loss": 0.015682760626077652, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.01581, "step": 124, "tokens/total": 3998280, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 56404} +{"epoch": 0.48828125, "grad_norm": 0.3201950192451477, "learning_rate": 9.113082447632394e-05, "loss": 0.016044847667217255, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.01617, "step": 125, "tokens/total": 4029616, "tokens/train_per_sec_per_gpu": 10.6, "tokens/trainable": 56878} +{"epoch": 0.4921875, "grad_norm": 0.5418305397033691, "learning_rate": 9.09570303250602e-05, "loss": 0.011210018768906593, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.01127, "step": 126, "tokens/total": 4060368, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 57362} +{"epoch": 0.49609375, "grad_norm": 0.46658676862716675, "learning_rate": 9.078173985499394e-05, "loss": 0.011282062157988548, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.01135, "step": 127, "tokens/total": 4089688, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 57792} +{"epoch": 0.5, "grad_norm": 0.609747052192688, "learning_rate": 9.060496036067713e-05, "loss": 0.008164255879819393, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.0082, "step": 128, "tokens/total": 4125048, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 58216} +{"epoch": 0.50390625, "grad_norm": 0.5438957214355469, "learning_rate": 9.042669919862615e-05, "loss": 0.008718976750969887, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00876, "step": 129, "tokens/total": 4156488, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 58680} +{"epoch": 0.5078125, "grad_norm": 0.4155794084072113, "learning_rate": 9.024696378701557e-05, "loss": 0.009099313989281654, "memory/device_reserved (GiB)": 66.81, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00914, "step": 130, "tokens/total": 4189344, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 59113} +{"epoch": 0.51171875, "grad_norm": 0.1384526491165161, "learning_rate": 9.006576160536948e-05, "loss": 0.0037914030253887177, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.0038, "step": 131, "tokens/total": 4220768, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 59562} +{"epoch": 0.515625, "grad_norm": 0.3150303065776825, "learning_rate": 8.988310019425035e-05, "loss": 0.0046677011996507645, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00468, "step": 132, "tokens/total": 4256880, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 60006} +{"epoch": 0.51953125, "grad_norm": 0.26445305347442627, "learning_rate": 8.969898715494506e-05, "loss": 0.0035425331443548203, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00355, "step": 133, "tokens/total": 4290928, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 60487} +{"epoch": 0.5234375, "grad_norm": 0.3066495656967163, "learning_rate": 8.951343014914869e-05, "loss": 0.003829283406957984, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00384, "step": 134, "tokens/total": 4325728, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 60953} +{"epoch": 0.52734375, "grad_norm": 0.4835602045059204, "learning_rate": 8.932643689864568e-05, "loss": 0.015325857326388359, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.01544, "step": 135, "tokens/total": 4358848, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 61440} +{"epoch": 0.53125, "grad_norm": 2.6733546257019043, "learning_rate": 8.913801518498845e-05, "loss": 0.00032594980439171195, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00033, "step": 136, "tokens/total": 4393504, "tokens/train_per_sec_per_gpu": 6.97, "tokens/trainable": 61901} +{"epoch": 0.53515625, "grad_norm": 0.7027373313903809, "learning_rate": 8.894817284917364e-05, "loss": 0.007671533618122339, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.0077, "step": 137, "tokens/total": 4427912, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 62355} +{"epoch": 0.5390625, "grad_norm": 0.3492235541343689, "learning_rate": 8.875691779131569e-05, "loss": 0.0021122631151229143, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00211, "step": 138, "tokens/total": 4462448, "tokens/train_per_sec_per_gpu": 6.45, "tokens/trainable": 62786} +{"epoch": 0.54296875, "grad_norm": 1.0635865926742554, "learning_rate": 8.856425797031829e-05, "loss": 0.007341798860579729, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00737, "step": 139, "tokens/total": 4495992, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 63228} +{"epoch": 0.546875, "grad_norm": 0.25408053398132324, "learning_rate": 8.837020140354295e-05, "loss": 0.0015857755206525326, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00159, "step": 140, "tokens/total": 4529600, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 63693} +{"epoch": 0.55078125, "grad_norm": 1.906456708908081, "learning_rate": 8.817475616647554e-05, "loss": 0.0033452173229306936, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00335, "step": 141, "tokens/total": 4562536, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 64159} +{"epoch": 0.5546875, "grad_norm": 0.20853380858898163, "learning_rate": 8.797793039239017e-05, "loss": 0.0007929815328679979, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00079, "step": 142, "tokens/total": 4595232, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 64608} +{"epoch": 0.55859375, "grad_norm": 0.7946325540542603, "learning_rate": 8.777973227201069e-05, "loss": 0.025885291397571564, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.02622, "step": 143, "tokens/total": 4629712, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 65117} +{"epoch": 0.5625, "grad_norm": 1.1202653646469116, "learning_rate": 8.758017005316988e-05, "loss": 0.02913888730108738, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.02957, "step": 144, "tokens/total": 4661752, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 65542} +{"epoch": 0.56640625, "grad_norm": 0.8849855065345764, "learning_rate": 8.737925204046629e-05, "loss": 0.03291400149464607, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.03346, "step": 145, "tokens/total": 4694416, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 65992} +{"epoch": 0.5703125, "grad_norm": 0.2878081798553467, "learning_rate": 8.717698659491851e-05, "loss": 0.003751722862944007, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00376, "step": 146, "tokens/total": 4728264, "tokens/train_per_sec_per_gpu": 7.08, "tokens/trainable": 66449} +{"epoch": 0.57421875, "grad_norm": 1.0328809022903442, "learning_rate": 8.697338213361735e-05, "loss": 0.008405243046581745, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00844, "step": 147, "tokens/total": 4758952, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 66897} +{"epoch": 0.578125, "grad_norm": 0.3584575057029724, "learning_rate": 8.676844712937552e-05, "loss": 0.015242046676576138, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.01536, "step": 148, "tokens/total": 4795624, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 67369} +{"epoch": 0.58203125, "grad_norm": 0.45859453082084656, "learning_rate": 8.656219011037509e-05, "loss": 0.020591434091329575, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.0208, "step": 149, "tokens/total": 4828976, "tokens/train_per_sec_per_gpu": 6.67, "tokens/trainable": 67857} +{"epoch": 0.5859375, "grad_norm": 0.1952313333749771, "learning_rate": 8.63546196598125e-05, "loss": 0.008346663787961006, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00838, "step": 150, "tokens/total": 4862128, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 68305} +{"epoch": 0.58984375, "grad_norm": 0.37953105568885803, "learning_rate": 8.614574441554145e-05, "loss": 0.012581082992255688, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.01266, "step": 151, "tokens/total": 4892760, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 68730} +{"epoch": 0.59375, "grad_norm": 1.1688644886016846, "learning_rate": 8.593557306971349e-05, "loss": 0.010865644551813602, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.01092, "step": 152, "tokens/total": 4925296, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 69185} +{"epoch": 0.59765625, "grad_norm": 0.1320619136095047, "learning_rate": 8.572411436841618e-05, "loss": 0.004878372419625521, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00489, "step": 153, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 69660} +{"epoch": 0.6015625, "grad_norm": 0.1546584814786911, "learning_rate": 8.551137711130922e-05, "loss": 0.004826843738555908, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00484, "step": 154, "tokens/total": 4985520, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 70066} +{"epoch": 0.60546875, "grad_norm": 0.507297158241272, "learning_rate": 8.529737015125824e-05, "loss": 0.009689124301075935, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00974, "step": 155, "tokens/total": 5019320, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 70532} +{"epoch": 0.609375, "grad_norm": 0.3594611585140228, "learning_rate": 8.508210239396639e-05, "loss": 0.008920657448470592, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00896, "step": 156, "tokens/total": 5052056, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 70981} +{"epoch": 0.61328125, "grad_norm": 0.3343689739704132, "learning_rate": 8.486558279760375e-05, "loss": 0.00623105326667428, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00625, "step": 157, "tokens/total": 5082280, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 71407} +{"epoch": 0.6171875, "grad_norm": 0.33906200528144836, "learning_rate": 8.464782037243449e-05, "loss": 0.004257656633853912, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00427, "step": 158, "tokens/total": 5113912, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 71848} +{"epoch": 0.62109375, "grad_norm": 0.46777060627937317, "learning_rate": 8.442882418044202e-05, "loss": 0.006453345064073801, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00647, "step": 159, "tokens/total": 5145200, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 72325} +{"epoch": 0.625, "grad_norm": 0.3066774308681488, "learning_rate": 8.420860333495179e-05, "loss": 0.005033606663346291, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00505, "step": 160, "tokens/total": 5174928, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 72739} +{"epoch": 0.62890625, "grad_norm": 0.5267209410667419, "learning_rate": 8.398716700025208e-05, "loss": 0.0011660713935270905, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00117, "step": 161, "tokens/total": 5206888, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 73191} +{"epoch": 0.6328125, "grad_norm": 0.8430089354515076, "learning_rate": 8.376452439121266e-05, "loss": 0.012451148591935635, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.01253, "step": 162, "tokens/total": 5236336, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 73642} +{"epoch": 0.63671875, "grad_norm": 0.3740254044532776, "learning_rate": 8.354068477290124e-05, "loss": 0.003140762448310852, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00315, "step": 163, "tokens/total": 5267760, "tokens/train_per_sec_per_gpu": 10.24, "tokens/trainable": 74101} +{"epoch": 0.640625, "grad_norm": 2.8422622680664062, "learning_rate": 8.331565746019807e-05, "loss": 0.01789935678243637, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.01806, "step": 164, "tokens/total": 5298896, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 74558} +{"epoch": 0.64453125, "grad_norm": 1.0752054452896118, "learning_rate": 8.308945181740812e-05, "loss": 0.037662606686353683, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.03838, "step": 165, "tokens/total": 5330792, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 75000} +{"epoch": 0.6484375, "grad_norm": 0.6339538097381592, "learning_rate": 8.286207725787153e-05, "loss": 0.00959011074155569, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00964, "step": 166, "tokens/total": 5364144, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 75445} +{"epoch": 0.65234375, "grad_norm": 0.9589905142784119, "learning_rate": 8.263354324357182e-05, "loss": 0.009931176900863647, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00998, "step": 167, "tokens/total": 5396656, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 75880} +{"epoch": 0.65625, "grad_norm": 1.7879951000213623, "learning_rate": 8.240385928474219e-05, "loss": 0.013965037651360035, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.01406, "step": 168, "tokens/total": 5427736, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 76294} +{"epoch": 0.66015625, "grad_norm": 0.20341947674751282, "learning_rate": 8.217303493946967e-05, "loss": 0.002526079071685672, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00253, "step": 169, "tokens/total": 5461752, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 76742} +{"epoch": 0.6640625, "grad_norm": 0.936194121837616, "learning_rate": 8.194107981329746e-05, "loss": 0.019525127485394478, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01972, "step": 170, "tokens/total": 5498688, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 77196} +{"epoch": 0.66796875, "grad_norm": 0.1100342720746994, "learning_rate": 8.170800355882518e-05, "loss": 0.0020719149615615606, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00207, "step": 171, "tokens/total": 5530936, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 77663} +{"epoch": 0.671875, "grad_norm": 0.24633842706680298, "learning_rate": 8.147381587530713e-05, "loss": 0.007016328163444996, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00704, "step": 172, "tokens/total": 5564176, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 78117} +{"epoch": 0.67578125, "grad_norm": 4.5535125732421875, "learning_rate": 8.123852650824877e-05, "loss": 0.010204946622252464, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.01026, "step": 173, "tokens/total": 5596856, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 78601} +{"epoch": 0.6796875, "grad_norm": 0.9343946576118469, "learning_rate": 8.100214524900103e-05, "loss": 0.026991577818989754, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.02736, "step": 174, "tokens/total": 5630016, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 79032} +{"epoch": 0.68359375, "grad_norm": 0.3405883014202118, "learning_rate": 8.076468193435301e-05, "loss": 0.006285574287176132, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00631, "step": 175, "tokens/total": 5659656, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 79456} +{"epoch": 0.6875, "grad_norm": 0.28939974308013916, "learning_rate": 8.052614644612253e-05, "loss": 0.01113656535744667, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.0112, "step": 176, "tokens/total": 5692408, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 79917} +{"epoch": 0.69140625, "grad_norm": 0.16163340210914612, "learning_rate": 8.028654871074489e-05, "loss": 0.004756147041916847, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00477, "step": 177, "tokens/total": 5724672, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 80358} +{"epoch": 0.6953125, "grad_norm": 0.12573541700839996, "learning_rate": 8.004589869885986e-05, "loss": 0.0036332590971142054, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00364, "step": 178, "tokens/total": 5754384, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 80777} +{"epoch": 0.69921875, "grad_norm": 0.5457701683044434, "learning_rate": 7.980420642489674e-05, "loss": 0.013721765950322151, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01382, "step": 179, "tokens/total": 5789344, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 81250} +{"epoch": 0.703125, "grad_norm": 0.08621444553136826, "learning_rate": 7.95614819466576e-05, "loss": 0.002031763317063451, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00203, "step": 180, "tokens/total": 5821816, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 81715} +{"epoch": 0.70703125, "grad_norm": 0.3252187669277191, "learning_rate": 7.931773536489872e-05, "loss": 0.008606836199760437, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00864, "step": 181, "tokens/total": 5854168, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 82160} +{"epoch": 0.7109375, "grad_norm": 0.13892722129821777, "learning_rate": 7.907297682291035e-05, "loss": 0.002676718169823289, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00268, "step": 182, "tokens/total": 5886064, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 82624} +{"epoch": 0.71484375, "grad_norm": 0.16093358397483826, "learning_rate": 7.882721650609442e-05, "loss": 0.003591650864109397, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.0036, "step": 183, "tokens/total": 5921648, "tokens/train_per_sec_per_gpu": 6.77, "tokens/trainable": 83125} +{"epoch": 0.71875, "grad_norm": 0.18848228454589844, "learning_rate": 7.85804646415409e-05, "loss": 0.0032373799476772547, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.00324, "step": 184, "tokens/total": 5954952, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 83581} +{"epoch": 0.72265625, "grad_norm": 0.4466186463832855, "learning_rate": 7.833273149760207e-05, "loss": 0.0065664807334542274, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00659, "step": 185, "tokens/total": 5985432, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 84051} +{"epoch": 0.7265625, "grad_norm": 0.09522125869989395, "learning_rate": 7.808402738346527e-05, "loss": 0.0010756568517535925, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00108, "step": 186, "tokens/total": 6020664, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 84500} +{"epoch": 0.73046875, "grad_norm": 0.07452689856290817, "learning_rate": 7.783436264872382e-05, "loss": 0.00025622325483709574, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00026, "step": 187, "tokens/total": 6055024, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 84986} +{"epoch": 0.734375, "grad_norm": 0.05060141906142235, "learning_rate": 7.758374768294647e-05, "loss": 0.0004705416795331985, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00047, "step": 188, "tokens/total": 6088296, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 85465} +{"epoch": 0.73828125, "grad_norm": 0.10166523605585098, "learning_rate": 7.733219291524489e-05, "loss": 0.0012232596054673195, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00122, "step": 189, "tokens/total": 6119840, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 85896} +{"epoch": 0.7421875, "grad_norm": 0.8172529339790344, "learning_rate": 7.707970881383977e-05, "loss": 0.016465138643980026, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.0166, "step": 190, "tokens/total": 6149880, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 86346} +{"epoch": 0.74609375, "grad_norm": 0.8233495354652405, "learning_rate": 7.682630588562518e-05, "loss": 0.015631692484021187, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01575, "step": 191, "tokens/total": 6182904, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 86777} +{"epoch": 0.75, "grad_norm": 0.4571199119091034, "learning_rate": 7.657199467573129e-05, "loss": 0.01844928041100502, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01862, "step": 192, "tokens/total": 6214144, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 87249} +{"epoch": 0.75390625, "grad_norm": 0.12050449103116989, "learning_rate": 7.631678576708561e-05, "loss": 0.0013220207765698433, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00132, "step": 193, "tokens/total": 6247072, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 87741} +{"epoch": 0.7578125, "grad_norm": 0.1994190365076065, "learning_rate": 7.606068977997255e-05, "loss": 0.0026756750885397196, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00268, "step": 194, "tokens/total": 6281112, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 88219} +{"epoch": 0.76171875, "grad_norm": 1.8829238414764404, "learning_rate": 7.580371737159148e-05, "loss": 0.00415714830160141, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00417, "step": 195, "tokens/total": 6310200, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 88630} +{"epoch": 0.765625, "grad_norm": 0.08868960291147232, "learning_rate": 7.554587923561324e-05, "loss": 0.0010223014978691936, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00102, "step": 196, "tokens/total": 6341792, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 89068} +{"epoch": 0.76953125, "grad_norm": 1.337791919708252, "learning_rate": 7.528718610173511e-05, "loss": 0.03097902610898018, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.03146, "step": 197, "tokens/total": 6374040, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 89489} +{"epoch": 0.7734375, "grad_norm": 0.34338176250457764, "learning_rate": 7.502764873523431e-05, "loss": 0.003578309901058674, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00358, "step": 198, "tokens/total": 6405552, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 89928} +{"epoch": 0.77734375, "grad_norm": 1.3755732774734497, "learning_rate": 7.476727793652011e-05, "loss": 0.01917838677763939, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.01936, "step": 199, "tokens/total": 6436016, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 90407} +{"epoch": 0.78125, "grad_norm": 0.3008771240711212, "learning_rate": 7.450608454068415e-05, "loss": 0.008290615864098072, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00833, "step": 200, "tokens/total": 6468656, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 90849} +{"epoch": 0.78515625, "grad_norm": 0.7008926272392273, "learning_rate": 7.424407941704987e-05, "loss": 0.011072509922087193, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.01113, "step": 201, "tokens/total": 6502728, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 91350} +{"epoch": 0.7890625, "grad_norm": 0.550906777381897, "learning_rate": 7.398127346871986e-05, "loss": 0.021570228040218353, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.0218, "step": 202, "tokens/total": 6535616, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 91805} +{"epoch": 0.79296875, "grad_norm": 0.07501114904880524, "learning_rate": 7.371767763212238e-05, "loss": 0.0015206043608486652, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00152, "step": 203, "tokens/total": 6567088, "tokens/train_per_sec_per_gpu": 10.13, "tokens/trainable": 92258} +{"epoch": 0.796875, "grad_norm": 0.07312483340501785, "learning_rate": 7.345330287655617e-05, "loss": 0.0025215570349246264, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00252, "step": 204, "tokens/total": 6600808, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 92721} +{"epoch": 0.80078125, "grad_norm": 0.23146873712539673, "learning_rate": 7.31881602037339e-05, "loss": 0.00735751586034894, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.00738, "step": 205, "tokens/total": 6631384, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 93160} +{"epoch": 0.8046875, "grad_norm": 0.18633374571800232, "learning_rate": 7.29222606473245e-05, "loss": 0.0038188721518963575, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00383, "step": 206, "tokens/total": 6661888, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 93605} +{"epoch": 0.80859375, "grad_norm": 0.22714412212371826, "learning_rate": 7.265561527249383e-05, "loss": 0.00824862439185381, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00828, "step": 207, "tokens/total": 6693456, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 94041} +{"epoch": 0.8125, "grad_norm": 0.27168479561805725, "learning_rate": 7.238823517544436e-05, "loss": 0.007973825559020042, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00801, "step": 208, "tokens/total": 6727840, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 94512} +{"epoch": 0.81640625, "grad_norm": 0.13765406608581543, "learning_rate": 7.212013148295333e-05, "loss": 0.003317681374028325, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00332, "step": 209, "tokens/total": 6761704, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 94994} +{"epoch": 0.8203125, "grad_norm": 0.2071571797132492, "learning_rate": 7.185131535190975e-05, "loss": 0.003561510005965829, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00357, "step": 210, "tokens/total": 6792648, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 95503} +{"epoch": 0.82421875, "grad_norm": 0.05963495746254921, "learning_rate": 7.158179796885005e-05, "loss": 0.0009822926949709654, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00098, "step": 211, "tokens/total": 6824424, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 95954} +{"epoch": 0.828125, "grad_norm": 0.2298942506313324, "learning_rate": 7.131159054949273e-05, "loss": 0.0012472084490582347, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00125, "step": 212, "tokens/total": 6861216, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 96428} +{"epoch": 0.83203125, "grad_norm": 0.0753602460026741, "learning_rate": 7.104070433827139e-05, "loss": 0.0009473812533542514, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00095, "step": 213, "tokens/total": 6892704, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 96828} +{"epoch": 0.8359375, "grad_norm": 0.2578119933605194, "learning_rate": 7.076915060786705e-05, "loss": 0.0031657726503908634, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.00317, "step": 214, "tokens/total": 6924816, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 97270} +{"epoch": 0.83984375, "grad_norm": 0.6233382225036621, "learning_rate": 7.049694065873882e-05, "loss": 0.015472324565052986, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.01559, "step": 215, "tokens/total": 6957536, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 97727} +{"epoch": 0.84375, "grad_norm": 0.52605801820755, "learning_rate": 7.022408581865382e-05, "loss": 0.010571413673460484, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.01063, "step": 216, "tokens/total": 6991448, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 98201} +{"epoch": 0.84765625, "grad_norm": 0.011538648046553135, "learning_rate": 6.99505974422157e-05, "loss": 0.00010628669406287372, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00011, "step": 217, "tokens/total": 7023712, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 98674} +{"epoch": 0.8515625, "grad_norm": 0.03689442202448845, "learning_rate": 6.967648691039213e-05, "loss": 0.0002904617867898196, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00029, "step": 218, "tokens/total": 7055040, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 99108} +{"epoch": 0.85546875, "grad_norm": 0.19775742292404175, "learning_rate": 6.940176563004123e-05, "loss": 0.0006396263488568366, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00064, "step": 219, "tokens/total": 7084944, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 99540} +{"epoch": 0.859375, "grad_norm": 0.2429797202348709, "learning_rate": 6.912644503343682e-05, "loss": 0.004035842139273882, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00404, "step": 220, "tokens/total": 7119264, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 99978} +{"epoch": 0.86328125, "grad_norm": 0.2705444395542145, "learning_rate": 6.885053657779273e-05, "loss": 0.0036566974595189095, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.00366, "step": 221, "tokens/total": 7150848, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 100450} +{"epoch": 0.8671875, "grad_norm": 0.34613147377967834, "learning_rate": 6.857405174478604e-05, "loss": 0.005163529887795448, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00518, "step": 222, "tokens/total": 7185176, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 100903} +{"epoch": 0.87109375, "grad_norm": 0.2595020532608032, "learning_rate": 6.82970020400792e-05, "loss": 0.003275419818237424, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00328, "step": 223, "tokens/total": 7219024, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 101351} +{"epoch": 0.875, "grad_norm": 0.3081379532814026, "learning_rate": 6.801939899284132e-05, "loss": 0.005343872122466564, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00536, "step": 224, "tokens/total": 7253816, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 101857} +{"epoch": 0.87890625, "grad_norm": 0.5749911069869995, "learning_rate": 6.774125415526827e-05, "loss": 0.0008614211110398173, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00086, "step": 225, "tokens/total": 7286344, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 102328} +{"epoch": 0.8828125, "grad_norm": 0.09437374770641327, "learning_rate": 6.746257910210214e-05, "loss": 0.001173520926386118, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00117, "step": 226, "tokens/total": 7317952, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 102779} +{"epoch": 0.88671875, "grad_norm": 0.47283485531806946, "learning_rate": 6.718338543014937e-05, "loss": 0.010553304105997086, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.01061, "step": 227, "tokens/total": 7347336, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 103228} +{"epoch": 0.890625, "grad_norm": 0.024763496592640877, "learning_rate": 6.69036847577983e-05, "loss": 0.0002379148791078478, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00024, "step": 228, "tokens/total": 7377592, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 103684} +{"epoch": 0.89453125, "grad_norm": 0.11814738810062408, "learning_rate": 6.662348872453553e-05, "loss": 0.0016575884073972702, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00166, "step": 229, "tokens/total": 7410184, "tokens/train_per_sec_per_gpu": 10.55, "tokens/trainable": 104145} +{"epoch": 0.8984375, "grad_norm": 0.045499883592128754, "learning_rate": 6.63428089904618e-05, "loss": 0.00011133735824842006, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00011, "step": 230, "tokens/total": 7443984, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 104598} +{"epoch": 0.90234375, "grad_norm": 0.3401120901107788, "learning_rate": 6.60616572358065e-05, "loss": 0.0031359237618744373, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00314, "step": 231, "tokens/total": 7472480, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 104998} +{"epoch": 0.90625, "grad_norm": 0.39189451932907104, "learning_rate": 6.578004516044172e-05, "loss": 0.003867883700877428, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00388, "step": 232, "tokens/total": 7505992, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 105465} +{"epoch": 0.91015625, "grad_norm": 0.2387831211090088, "learning_rate": 6.549798448339548e-05, "loss": 0.005181013140827417, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00519, "step": 233, "tokens/total": 7541168, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 105928} +{"epoch": 0.9140625, "grad_norm": 0.032855600118637085, "learning_rate": 6.521548694236384e-05, "loss": 0.0003232336894143373, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00032, "step": 234, "tokens/total": 7574208, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 106365} +{"epoch": 0.91796875, "grad_norm": 0.4487527906894684, "learning_rate": 6.493256429322259e-05, "loss": 0.003892584703862667, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.0039, "step": 235, "tokens/total": 7607016, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 106796} +{"epoch": 0.921875, "grad_norm": 0.09075319021940231, "learning_rate": 6.464922830953799e-05, "loss": 0.0004531170998234302, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00045, "step": 236, "tokens/total": 7641744, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 107227} +{"epoch": 0.92578125, "grad_norm": 0.9350703954696655, "learning_rate": 6.436549078207688e-05, "loss": 0.008091854862868786, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00812, "step": 237, "tokens/total": 7673096, "tokens/train_per_sec_per_gpu": 10.17, "tokens/trainable": 107687} +{"epoch": 0.9296875, "grad_norm": 0.3105892539024353, "learning_rate": 6.408136351831592e-05, "loss": 0.001955241896212101, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00196, "step": 238, "tokens/total": 7705344, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 108127} +{"epoch": 0.93359375, "grad_norm": 0.33667632937431335, "learning_rate": 6.379685834195036e-05, "loss": 0.0013365292688831687, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00134, "step": 239, "tokens/total": 7740240, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 108633} +{"epoch": 0.9375, "grad_norm": 0.3038717806339264, "learning_rate": 6.351198709240186e-05, "loss": 0.0021812135819345713, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00218, "step": 240, "tokens/total": 7775392, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 109085} +{"epoch": 0.94140625, "grad_norm": 0.5131248831748962, "learning_rate": 6.32267616243259e-05, "loss": 0.007670596241950989, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.0077, "step": 241, "tokens/total": 7807296, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 109548} +{"epoch": 0.9453125, "grad_norm": 0.026113178580999374, "learning_rate": 6.294119380711849e-05, "loss": 0.0002669589885044843, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00027, "step": 242, "tokens/total": 7838200, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 109975} +{"epoch": 0.94921875, "grad_norm": 5.881136894226074, "learning_rate": 6.265529552442209e-05, "loss": 0.020608944818377495, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.02082, "step": 243, "tokens/total": 7870936, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 110436} +{"epoch": 0.953125, "grad_norm": 0.0905216857790947, "learning_rate": 6.236907867363127e-05, "loss": 0.0005413979524746537, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00054, "step": 244, "tokens/total": 7902512, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 110882} +{"epoch": 0.95703125, "grad_norm": 0.19139164686203003, "learning_rate": 6.208255516539749e-05, "loss": 0.0020854012109339237, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00209, "step": 245, "tokens/total": 7933224, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 111355} +{"epoch": 0.9609375, "grad_norm": 0.3969123661518097, "learning_rate": 6.179573692313344e-05, "loss": 0.001393370795994997, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00139, "step": 246, "tokens/total": 7968456, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 111803} +{"epoch": 0.96484375, "grad_norm": 0.07306085526943207, "learning_rate": 6.150863588251694e-05, "loss": 0.0004412179405335337, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00044, "step": 247, "tokens/total": 8003592, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 112270} +{"epoch": 0.96875, "grad_norm": 0.05681666359305382, "learning_rate": 6.122126399099419e-05, "loss": 0.0006051425589248538, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00061, "step": 248, "tokens/total": 8040048, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 112778} +{"epoch": 0.97265625, "grad_norm": 0.6025075912475586, "learning_rate": 6.0933633207282615e-05, "loss": 0.014106284826993942, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.01421, "step": 249, "tokens/total": 8074000, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 113284} +{"epoch": 0.9765625, "grad_norm": 0.15296000242233276, "learning_rate": 6.064575550087316e-05, "loss": 0.0019644980784505606, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00197, "step": 250, "tokens/total": 8108304, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 113722} +{"epoch": 0.98046875, "grad_norm": 0.3111872971057892, "learning_rate": 6.0357642851532245e-05, "loss": 0.0033065672032535076, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00331, "step": 251, "tokens/total": 8143464, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 114213} +{"epoch": 0.984375, "grad_norm": 0.012475071474909782, "learning_rate": 6.0069307248803294e-05, "loss": 0.00014538958203047514, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00015, "step": 252, "tokens/total": 8175576, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 114634} +{"epoch": 0.98828125, "grad_norm": 0.4037182033061981, "learning_rate": 5.9780760691507635e-05, "loss": 0.013668760657310486, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.01376, "step": 253, "tokens/total": 8205528, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 115066} +{"epoch": 0.9921875, "grad_norm": 0.23934832215309143, "learning_rate": 5.9492015187245334e-05, "loss": 0.004441256169229746, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00445, "step": 254, "tokens/total": 8239424, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 115520} +{"epoch": 0.99609375, "grad_norm": 0.3984468877315521, "learning_rate": 5.920308275189541e-05, "loss": 0.011719046160578728, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.01179, "step": 255, "tokens/total": 8270392, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 115921} +{"epoch": 1.0, "grad_norm": 0.04123145341873169, "learning_rate": 5.8913975409115874e-05, "loss": 0.00042441050754860044, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00042, "step": 256, "tokens/total": 8302096, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 116384} +{"epoch": 1.00390625, "grad_norm": 0.16751927137374878, "learning_rate": 5.8624705189843395e-05, "loss": 0.00269941589795053, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.0027, "step": 257, "tokens/total": 8337064, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 116871} +{"epoch": 1.0078125, "grad_norm": 0.039524275809526443, "learning_rate": 5.833528413179249e-05, "loss": 0.0005765279638580978, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00058, "step": 258, "tokens/total": 8372840, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 117369} +{"epoch": 1.01171875, "grad_norm": 0.3743211627006531, "learning_rate": 5.80457242789548e-05, "loss": 0.004244463983923197, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00425, "step": 259, "tokens/total": 8409728, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 117866} +{"epoch": 1.015625, "grad_norm": 0.36296913027763367, "learning_rate": 5.77560376810977e-05, "loss": 0.018455233424901962, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.01863, "step": 260, "tokens/total": 8442296, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 118330} +{"epoch": 1.01953125, "grad_norm": 0.03827071562409401, "learning_rate": 5.7466236393263005e-05, "loss": 0.0006370480987243354, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00064, "step": 261, "tokens/total": 8474568, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 118776} +{"epoch": 1.0234375, "grad_norm": 0.0455787293612957, "learning_rate": 5.717633247526522e-05, "loss": 0.0009987982921302319, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.001, "step": 262, "tokens/total": 8505520, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 119211} +{"epoch": 1.02734375, "grad_norm": 1.4428249597549438, "learning_rate": 5.688633799118971e-05, "loss": 0.01868467405438423, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.01886, "step": 263, "tokens/total": 8535064, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 119616} +{"epoch": 1.03125, "grad_norm": 0.32949331402778625, "learning_rate": 5.659626500889066e-05, "loss": 0.004380368161946535, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00439, "step": 264, "tokens/total": 8563600, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 120086} +{"epoch": 1.03515625, "grad_norm": 0.05385122448205948, "learning_rate": 5.6306125599488905e-05, "loss": 0.0010576549684628844, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00106, "step": 265, "tokens/total": 8592648, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 120525} +{"epoch": 1.0390625, "grad_norm": 0.3184790015220642, "learning_rate": 5.601593183686955e-05, "loss": 0.002495346823707223, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.0025, "step": 266, "tokens/total": 8627328, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 121001} +{"epoch": 1.04296875, "grad_norm": 0.18995647132396698, "learning_rate": 5.572569579717961e-05, "loss": 0.006274753250181675, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00629, "step": 267, "tokens/total": 8659760, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 121465} +{"epoch": 1.046875, "grad_norm": 0.053728051483631134, "learning_rate": 5.543542955832538e-05, "loss": 0.000970518565736711, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00097, "step": 268, "tokens/total": 8696968, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 121908} +{"epoch": 1.05078125, "grad_norm": 0.442041277885437, "learning_rate": 5.514514519946986e-05, "loss": 0.014407705515623093, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01451, "step": 269, "tokens/total": 8735392, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 122406} +{"epoch": 1.0546875, "grad_norm": 0.04578741639852524, "learning_rate": 5.485485480053015e-05, "loss": 0.0010556461056694388, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00106, "step": 270, "tokens/total": 8769840, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 122812} +{"epoch": 1.05859375, "grad_norm": 0.05678270757198334, "learning_rate": 5.4564570441674645e-05, "loss": 0.0010693976655602455, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00107, "step": 271, "tokens/total": 8800648, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 123252} +{"epoch": 1.0625, "grad_norm": 0.10525481402873993, "learning_rate": 5.42743042028204e-05, "loss": 0.0030147028155624866, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00302, "step": 272, "tokens/total": 8832480, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 123692} +{"epoch": 1.06640625, "grad_norm": 0.11729303747415543, "learning_rate": 5.3984068163130464e-05, "loss": 0.003172355704009533, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00318, "step": 273, "tokens/total": 8864840, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 124162} +{"epoch": 1.0703125, "grad_norm": 0.14679564535617828, "learning_rate": 5.369387440051111e-05, "loss": 0.002885996364057064, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00289, "step": 274, "tokens/total": 8897272, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 124625} +{"epoch": 1.07421875, "grad_norm": 0.11044740676879883, "learning_rate": 5.340373499110935e-05, "loss": 0.001313705462962389, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00131, "step": 275, "tokens/total": 8930688, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 125119} +{"epoch": 1.078125, "grad_norm": 0.156024768948555, "learning_rate": 5.3113662008810304e-05, "loss": 0.0010021356865763664, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.001, "step": 276, "tokens/total": 8967216, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 125601} +{"epoch": 1.08203125, "grad_norm": 0.1664089858531952, "learning_rate": 5.282366752473479e-05, "loss": 0.00420245248824358, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00421, "step": 277, "tokens/total": 9001368, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 126075} +{"epoch": 1.0859375, "grad_norm": 0.10609115660190582, "learning_rate": 5.2533763606737005e-05, "loss": 0.001969517907127738, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00197, "step": 278, "tokens/total": 9036808, "tokens/train_per_sec_per_gpu": 6.92, "tokens/trainable": 126491} +{"epoch": 1.08984375, "grad_norm": 0.030042298138141632, "learning_rate": 5.224396231890232e-05, "loss": 0.0007334762485697865, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00073, "step": 279, "tokens/total": 9067840, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 126940} +{"epoch": 1.09375, "grad_norm": 0.19773150980472565, "learning_rate": 5.195427572104522e-05, "loss": 0.004645208828151226, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00466, "step": 280, "tokens/total": 9098936, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 127376} +{"epoch": 1.09765625, "grad_norm": 0.06614639610052109, "learning_rate": 5.166471586820751e-05, "loss": 0.00115764990914613, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00116, "step": 281, "tokens/total": 9133264, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 127841} +{"epoch": 1.1015625, "grad_norm": 0.17236217856407166, "learning_rate": 5.1375294810156615e-05, "loss": 0.0030563180334866047, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00306, "step": 282, "tokens/total": 9165008, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 128288} +{"epoch": 1.10546875, "grad_norm": 0.09255858510732651, "learning_rate": 5.1086024590884144e-05, "loss": 0.001310216961428523, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00131, "step": 283, "tokens/total": 9197952, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 128737} +{"epoch": 1.109375, "grad_norm": 0.06139812991023064, "learning_rate": 5.079691724810461e-05, "loss": 0.0006206797552295029, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00062, "step": 284, "tokens/total": 9232576, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 129172} +{"epoch": 1.11328125, "grad_norm": 0.007052847649902105, "learning_rate": 5.0507984812754684e-05, "loss": 0.0001395267463522032, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00014, "step": 285, "tokens/total": 9264568, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 129669} +{"epoch": 1.1171875, "grad_norm": 0.031225448474287987, "learning_rate": 5.021923930849237e-05, "loss": 0.0002673775888979435, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00027, "step": 286, "tokens/total": 9296544, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 130155} +{"epoch": 1.12109375, "grad_norm": 0.1896362155675888, "learning_rate": 4.99306927511967e-05, "loss": 0.0016951241996139288, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.0017, "step": 287, "tokens/total": 9327936, "tokens/train_per_sec_per_gpu": 10.46, "tokens/trainable": 130595} +{"epoch": 1.125, "grad_norm": 0.006905039772391319, "learning_rate": 4.964235714846775e-05, "loss": 5.2819057600572705e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00005, "step": 288, "tokens/total": 9357984, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 131015} +{"epoch": 1.12890625, "grad_norm": 0.03182801976799965, "learning_rate": 4.9354244499126866e-05, "loss": 0.00017993932124227285, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00018, "step": 289, "tokens/total": 9388664, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 131481} +{"epoch": 1.1328125, "grad_norm": 0.009205094538629055, "learning_rate": 4.90663667927174e-05, "loss": 0.000146299964399077, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00015, "step": 290, "tokens/total": 9420504, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 131930} +{"epoch": 1.13671875, "grad_norm": 0.02586408145725727, "learning_rate": 4.877873600900581e-05, "loss": 0.0001247923937626183, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00012, "step": 291, "tokens/total": 9452664, "tokens/train_per_sec_per_gpu": 6.24, "tokens/trainable": 132388} +{"epoch": 1.140625, "grad_norm": 0.02640143409371376, "learning_rate": 4.849136411748306e-05, "loss": 0.00020097586093470454, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.0002, "step": 292, "tokens/total": 9488896, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 132860} +{"epoch": 1.14453125, "grad_norm": 0.002351253293454647, "learning_rate": 4.8204263076866574e-05, "loss": 3.1883093470241874e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00003, "step": 293, "tokens/total": 9522112, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 133360} +{"epoch": 1.1484375, "grad_norm": 0.005519477650523186, "learning_rate": 4.791744483460251e-05, "loss": 2.5682562409201637e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00003, "step": 294, "tokens/total": 9555632, "tokens/train_per_sec_per_gpu": 6.62, "tokens/trainable": 133793} +{"epoch": 1.15234375, "grad_norm": 0.015905005857348442, "learning_rate": 4.7630921326368736e-05, "loss": 0.00012417348625604063, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00012, "step": 295, "tokens/total": 9590904, "tokens/train_per_sec_per_gpu": 6.44, "tokens/trainable": 134218} +{"epoch": 1.15625, "grad_norm": 0.07638166844844818, "learning_rate": 4.7344704475577916e-05, "loss": 0.000370003777788952, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00037, "step": 296, "tokens/total": 9620288, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 134654} +{"epoch": 1.16015625, "grad_norm": 0.432271808385849, "learning_rate": 4.705880619288153e-05, "loss": 0.008437499403953552, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00847, "step": 297, "tokens/total": 9651984, "tokens/train_per_sec_per_gpu": 7.12, "tokens/trainable": 135056} +{"epoch": 1.1640625, "grad_norm": 0.021878046914935112, "learning_rate": 4.677323837567412e-05, "loss": 0.00024254585150629282, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00024, "step": 298, "tokens/total": 9681368, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 135483} +{"epoch": 1.16796875, "grad_norm": 0.010517086833715439, "learning_rate": 4.6488012907598146e-05, "loss": 8.78482242114842e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00009, "step": 299, "tokens/total": 9712872, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 135957} +{"epoch": 1.171875, "grad_norm": 0.044817838817834854, "learning_rate": 4.620314165804964e-05, "loss": 0.00019362995226401836, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00019, "step": 300, "tokens/total": 9745184, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 136430} +{"epoch": 1.17578125, "grad_norm": 0.03348878026008606, "learning_rate": 4.591863648168407e-05, "loss": 0.0001637180830584839, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00016, "step": 301, "tokens/total": 9778928, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 136896} +{"epoch": 1.1796875, "grad_norm": 0.03079141676425934, "learning_rate": 4.5634509217923135e-05, "loss": 0.00011745907977456227, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00012, "step": 302, "tokens/total": 9810752, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 137331} +{"epoch": 1.18359375, "grad_norm": 0.02545136772096157, "learning_rate": 4.535077169046201e-05, "loss": 0.00016438406601082534, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00016, "step": 303, "tokens/total": 9843672, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 137768} +{"epoch": 1.1875, "grad_norm": 0.12747670710086823, "learning_rate": 4.506743570677743e-05, "loss": 0.0009499803418293595, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00095, "step": 304, "tokens/total": 9873752, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 138200} +{"epoch": 1.19140625, "grad_norm": 0.3486821949481964, "learning_rate": 4.478451305763618e-05, "loss": 0.00109273346606642, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00109, "step": 305, "tokens/total": 9906272, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 138660} +{"epoch": 1.1953125, "grad_norm": 0.0003212794254068285, "learning_rate": 4.450201551660454e-05, "loss": 7.693727638979908e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00001, "step": 306, "tokens/total": 9936296, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 139092} +{"epoch": 1.19921875, "grad_norm": 0.4475044012069702, "learning_rate": 4.4219954839558276e-05, "loss": 0.0019680559635162354, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00197, "step": 307, "tokens/total": 9967720, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 139533} +{"epoch": 1.203125, "grad_norm": 0.2071646898984909, "learning_rate": 4.393834276419352e-05, "loss": 0.002072911709547043, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.00208, "step": 308, "tokens/total": 9997176, "tokens/train_per_sec_per_gpu": 10.84, "tokens/trainable": 140020} +{"epoch": 1.20703125, "grad_norm": 0.0011162722948938608, "learning_rate": 4.36571910095382e-05, "loss": 1.4463468687608838e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00001, "step": 309, "tokens/total": 10031872, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 140465} +{"epoch": 1.2109375, "grad_norm": 0.771913468837738, "learning_rate": 4.337651127546448e-05, "loss": 0.010370002128183842, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01042, "step": 310, "tokens/total": 10065240, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 140915} +{"epoch": 1.21484375, "grad_norm": 0.5210821628570557, "learning_rate": 4.3096315242201736e-05, "loss": 0.002128430875018239, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00213, "step": 311, "tokens/total": 10096640, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 141350} +{"epoch": 1.21875, "grad_norm": 0.002033350756391883, "learning_rate": 4.2816614569850635e-05, "loss": 1.8731478121480905e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00002, "step": 312, "tokens/total": 10130192, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 141789} +{"epoch": 1.22265625, "grad_norm": 0.8565779328346252, "learning_rate": 4.2537420897897864e-05, "loss": 0.004097561351954937, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00411, "step": 313, "tokens/total": 10164248, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 142266} +{"epoch": 1.2265625, "grad_norm": 0.4908657968044281, "learning_rate": 4.225874584473174e-05, "loss": 0.010432016104459763, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.01049, "step": 314, "tokens/total": 10194664, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 142704} +{"epoch": 1.23046875, "grad_norm": 0.010734555311501026, "learning_rate": 4.19806010071587e-05, "loss": 5.926289304625243e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00006, "step": 315, "tokens/total": 10227768, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 143168} +{"epoch": 1.234375, "grad_norm": 0.006337712984532118, "learning_rate": 4.170299795992081e-05, "loss": 4.593456469592638e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00005, "step": 316, "tokens/total": 10260464, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 143615} +{"epoch": 1.23828125, "grad_norm": 0.036356426775455475, "learning_rate": 4.142594825521398e-05, "loss": 0.00027569415397010744, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00028, "step": 317, "tokens/total": 10297528, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 144096} +{"epoch": 1.2421875, "grad_norm": 0.11962169408798218, "learning_rate": 4.114946342220728e-05, "loss": 0.0006143681821413338, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00061, "step": 318, "tokens/total": 10331448, "tokens/train_per_sec_per_gpu": 7.01, "tokens/trainable": 144543} +{"epoch": 1.24609375, "grad_norm": 0.15890391170978546, "learning_rate": 4.087355496656321e-05, "loss": 0.0011525916634127498, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00115, "step": 319, "tokens/total": 10363152, "tokens/train_per_sec_per_gpu": 10.19, "tokens/trainable": 145016} +{"epoch": 1.25, "grad_norm": 0.19057320058345795, "learning_rate": 4.05982343699588e-05, "loss": 0.001490987604483962, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00149, "step": 320, "tokens/total": 10397832, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 145511} +{"epoch": 1.25390625, "grad_norm": 0.15217840671539307, "learning_rate": 4.0323513089607876e-05, "loss": 0.001400192966684699, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.0014, "step": 321, "tokens/total": 10431416, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 145978} +{"epoch": 1.2578125, "grad_norm": 0.005817072466015816, "learning_rate": 4.004940255778431e-05, "loss": 6.796614616177976e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00007, "step": 322, "tokens/total": 10467384, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 146422} +{"epoch": 1.26171875, "grad_norm": 0.46233880519866943, "learning_rate": 3.977591418134619e-05, "loss": 0.0036468501202762127, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00365, "step": 323, "tokens/total": 10500792, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 146863} +{"epoch": 1.265625, "grad_norm": 0.6581670641899109, "learning_rate": 3.95030593412612e-05, "loss": 0.013498503714799881, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.01359, "step": 324, "tokens/total": 10532592, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 147286} +{"epoch": 1.26953125, "grad_norm": 1.7005128860473633, "learning_rate": 3.923084939213296e-05, "loss": 0.0044654919765889645, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00448, "step": 325, "tokens/total": 10562392, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 147708} +{"epoch": 1.2734375, "grad_norm": 0.022722577676177025, "learning_rate": 3.895929566172861e-05, "loss": 0.0001747789210639894, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00017, "step": 326, "tokens/total": 10595416, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 148172} +{"epoch": 1.27734375, "grad_norm": 0.10423026233911514, "learning_rate": 3.868840945050728e-05, "loss": 0.000686120823957026, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00069, "step": 327, "tokens/total": 10630504, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 148604} +{"epoch": 1.28125, "grad_norm": 0.10789469629526138, "learning_rate": 3.841820203114995e-05, "loss": 0.00040018599247559905, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.0004, "step": 328, "tokens/total": 10661632, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 149051} +{"epoch": 1.28515625, "grad_norm": 0.01782165840268135, "learning_rate": 3.814868464809027e-05, "loss": 0.00011322848877171054, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00011, "step": 329, "tokens/total": 10691544, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 149453} +{"epoch": 1.2890625, "grad_norm": 1.1146384477615356, "learning_rate": 3.787986851704667e-05, "loss": 0.003572220914065838, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00358, "step": 330, "tokens/total": 10724768, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 149883} +{"epoch": 1.29296875, "grad_norm": 0.01857539638876915, "learning_rate": 3.7611764824555654e-05, "loss": 0.0001594006025698036, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00016, "step": 331, "tokens/total": 10756008, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 150357} +{"epoch": 1.296875, "grad_norm": 0.4012798070907593, "learning_rate": 3.734438472750619e-05, "loss": 0.005096923094242811, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00511, "step": 332, "tokens/total": 10793152, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 150828} +{"epoch": 1.30078125, "grad_norm": 0.007219552528113127, "learning_rate": 3.707773935267552e-05, "loss": 9.181438508676365e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00009, "step": 333, "tokens/total": 10827624, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 151314} +{"epoch": 1.3046875, "grad_norm": 0.002470614155754447, "learning_rate": 3.68118397962661e-05, "loss": 5.4525913583347574e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00005, "step": 334, "tokens/total": 10859712, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 151722} +{"epoch": 1.30859375, "grad_norm": 0.5111032128334045, "learning_rate": 3.654669712344384e-05, "loss": 0.0032948090229183435, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.0033, "step": 335, "tokens/total": 10888792, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 152178} +{"epoch": 1.3125, "grad_norm": 0.009986547753214836, "learning_rate": 3.628232236787763e-05, "loss": 9.483825124334544e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00009, "step": 336, "tokens/total": 10922968, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 152617} +{"epoch": 1.31640625, "grad_norm": 0.011982148513197899, "learning_rate": 3.6018726531280144e-05, "loss": 8.829354192130268e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00009, "step": 337, "tokens/total": 10958000, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 153108} +{"epoch": 1.3203125, "grad_norm": 0.005752270575612783, "learning_rate": 3.575592058295017e-05, "loss": 7.349728548433632e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00007, "step": 338, "tokens/total": 10989872, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 153589} +{"epoch": 1.32421875, "grad_norm": 0.0030457100365310907, "learning_rate": 3.549391545931585e-05, "loss": 3.5914184991270304e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00004, "step": 339, "tokens/total": 11020784, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 154049} +{"epoch": 1.328125, "grad_norm": 0.005668785888701677, "learning_rate": 3.5232722063479914e-05, "loss": 9.668052371125668e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.0001, "step": 340, "tokens/total": 11053240, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 154498} +{"epoch": 1.33203125, "grad_norm": 0.32424047589302063, "learning_rate": 3.49723512647657e-05, "loss": 0.0001455733145121485, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00015, "step": 341, "tokens/total": 11089040, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 154998} +{"epoch": 1.3359375, "grad_norm": 0.446359783411026, "learning_rate": 3.471281389826491e-05, "loss": 0.0023226505145430565, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00233, "step": 342, "tokens/total": 11119672, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 155403} +{"epoch": 1.33984375, "grad_norm": 0.19236120581626892, "learning_rate": 3.4454120764386764e-05, "loss": 0.0016623834380879998, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00166, "step": 343, "tokens/total": 11155040, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 155858} +{"epoch": 1.34375, "grad_norm": 0.016125289723277092, "learning_rate": 3.4196282628408526e-05, "loss": 0.00019859177700709552, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.0002, "step": 344, "tokens/total": 11185952, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 156303} +{"epoch": 1.34765625, "grad_norm": 0.048251327127218246, "learning_rate": 3.3939310220027456e-05, "loss": 0.0004017760220449418, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.0004, "step": 345, "tokens/total": 11219272, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 156798} +{"epoch": 1.3515625, "grad_norm": 0.04502120614051819, "learning_rate": 3.3683214232914404e-05, "loss": 0.000406499340897426, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00041, "step": 346, "tokens/total": 11253424, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 157271} +{"epoch": 1.35546875, "grad_norm": 0.0421210415661335, "learning_rate": 3.342800532426873e-05, "loss": 0.0004367561195977032, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00044, "step": 347, "tokens/total": 11286216, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 157737} +{"epoch": 1.359375, "grad_norm": 0.27186524868011475, "learning_rate": 3.317369411437484e-05, "loss": 0.001962844515219331, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00196, "step": 348, "tokens/total": 11319480, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 158205} +{"epoch": 1.36328125, "grad_norm": 0.06002522632479668, "learning_rate": 3.292029118616024e-05, "loss": 0.00022508599795401096, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00023, "step": 349, "tokens/total": 11350472, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 158628} +{"epoch": 1.3671875, "grad_norm": 0.1289682686328888, "learning_rate": 3.266780708475511e-05, "loss": 0.0008346312679350376, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00083, "step": 350, "tokens/total": 11383088, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 159106} +{"epoch": 1.37109375, "grad_norm": 0.4021662473678589, "learning_rate": 3.241625231705354e-05, "loss": 0.001078076777048409, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00108, "step": 351, "tokens/total": 11418392, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 159565} +{"epoch": 1.375, "grad_norm": 0.007515177596360445, "learning_rate": 3.216563735127618e-05, "loss": 8.778866322245449e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00009, "step": 352, "tokens/total": 11449784, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 160042} +{"epoch": 1.37890625, "grad_norm": 0.10824093222618103, "learning_rate": 3.191597261653475e-05, "loss": 0.0008051986806094646, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00081, "step": 353, "tokens/total": 11484240, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 160502} +{"epoch": 1.3828125, "grad_norm": 0.20152361690998077, "learning_rate": 3.166726850239794e-05, "loss": 0.0013039528857916594, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.0013, "step": 354, "tokens/total": 11513360, "tokens/train_per_sec_per_gpu": 11.04, "tokens/trainable": 160950} +{"epoch": 1.38671875, "grad_norm": 0.012946153059601784, "learning_rate": 3.141953535845912e-05, "loss": 6.292920443229377e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00006, "step": 355, "tokens/total": 11543112, "tokens/train_per_sec_per_gpu": 10.31, "tokens/trainable": 161400} +{"epoch": 1.390625, "grad_norm": 0.018434816971421242, "learning_rate": 3.11727834939056e-05, "loss": 7.825673674233258e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00008, "step": 356, "tokens/total": 11573464, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 161863} +{"epoch": 1.39453125, "grad_norm": 0.07929867506027222, "learning_rate": 3.092702317708967e-05, "loss": 0.00024038039555307478, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00024, "step": 357, "tokens/total": 11604896, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 162319} +{"epoch": 1.3984375, "grad_norm": 0.009394684806466103, "learning_rate": 3.0682264635101276e-05, "loss": 7.56659937906079e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00008, "step": 358, "tokens/total": 11636368, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 162771} +{"epoch": 1.40234375, "grad_norm": 0.20029613375663757, "learning_rate": 3.0438518053342407e-05, "loss": 0.0011994667584076524, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.0012, "step": 359, "tokens/total": 11670216, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 163243} +{"epoch": 1.40625, "grad_norm": 0.004786157514899969, "learning_rate": 3.0195793575103266e-05, "loss": 6.977804150665179e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00007, "step": 360, "tokens/total": 11707816, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 163707} +{"epoch": 1.41015625, "grad_norm": 0.3029446005821228, "learning_rate": 2.9954101301140146e-05, "loss": 0.002137573901563883, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00214, "step": 361, "tokens/total": 11742584, "tokens/train_per_sec_per_gpu": 6.94, "tokens/trainable": 164157} +{"epoch": 1.4140625, "grad_norm": 0.1359102874994278, "learning_rate": 2.9713451289255123e-05, "loss": 0.0007604390266351402, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00076, "step": 362, "tokens/total": 11770384, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 164598} +{"epoch": 1.41796875, "grad_norm": 0.0026488504372537136, "learning_rate": 2.9473853553877484e-05, "loss": 3.7182420783210546e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00004, "step": 363, "tokens/total": 11803208, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 165072} +{"epoch": 1.421875, "grad_norm": 0.010626137256622314, "learning_rate": 2.9235318065647e-05, "loss": 6.85216800775379e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00007, "step": 364, "tokens/total": 11840480, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 165533} +{"epoch": 1.42578125, "grad_norm": 0.0028323912993073463, "learning_rate": 2.8997854750998964e-05, "loss": 3.113215643679723e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00003, "step": 365, "tokens/total": 11875544, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 165998} +{"epoch": 1.4296875, "grad_norm": 0.0012422986328601837, "learning_rate": 2.8761473491751258e-05, "loss": 1.6206762666115537e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00002, "step": 366, "tokens/total": 11907656, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 166433} +{"epoch": 1.43359375, "grad_norm": 0.0025064742658287287, "learning_rate": 2.8526184124692883e-05, "loss": 2.5169170839944854e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00003, "step": 367, "tokens/total": 11941904, "tokens/train_per_sec_per_gpu": 6.42, "tokens/trainable": 166871} +{"epoch": 1.4375, "grad_norm": 0.08601590245962143, "learning_rate": 2.829199644117484e-05, "loss": 0.0006193055887706578, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00062, "step": 368, "tokens/total": 11973624, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 167281} +{"epoch": 1.44140625, "grad_norm": 0.011195171624422073, "learning_rate": 2.8058920186702553e-05, "loss": 4.161614924669266e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00004, "step": 369, "tokens/total": 12010360, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 167754} +{"epoch": 1.4453125, "grad_norm": 0.07311386615037918, "learning_rate": 2.782696506053033e-05, "loss": 0.00030672139837406576, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00031, "step": 370, "tokens/total": 12044544, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 168186} +{"epoch": 1.44921875, "grad_norm": 0.66423100233078, "learning_rate": 2.7596140715257824e-05, "loss": 0.003235272830352187, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00324, "step": 371, "tokens/total": 12075792, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 168651} +{"epoch": 1.453125, "grad_norm": 0.002897960366681218, "learning_rate": 2.7366456756428184e-05, "loss": 1.6286274330923334e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00002, "step": 372, "tokens/total": 12108088, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 169108} +{"epoch": 1.45703125, "grad_norm": 0.007903343066573143, "learning_rate": 2.7137922742128486e-05, "loss": 5.8128731325268745e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00006, "step": 373, "tokens/total": 12140888, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 169607} +{"epoch": 1.4609375, "grad_norm": 0.002327305730432272, "learning_rate": 2.691054818259188e-05, "loss": 2.753439184743911e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00003, "step": 374, "tokens/total": 12174560, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 170080} +{"epoch": 1.46484375, "grad_norm": 0.01170456875115633, "learning_rate": 2.6684342539801933e-05, "loss": 0.00013670405314769596, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00014, "step": 375, "tokens/total": 12204848, "tokens/train_per_sec_per_gpu": 10.01, "tokens/trainable": 170532} +{"epoch": 1.46875, "grad_norm": 0.0016344421310350299, "learning_rate": 2.645931522709877e-05, "loss": 1.4089388969296124e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00001, "step": 376, "tokens/total": 12238472, "tokens/train_per_sec_per_gpu": 10.8, "tokens/trainable": 171012} +{"epoch": 1.47265625, "grad_norm": 0.010132583789527416, "learning_rate": 2.6235475608787365e-05, "loss": 8.691346738487482e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00009, "step": 377, "tokens/total": 12271024, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 171473} +{"epoch": 1.4765625, "grad_norm": 0.2952722907066345, "learning_rate": 2.6012832999747916e-05, "loss": 0.0012409158516675234, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00124, "step": 378, "tokens/total": 12301312, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 171920} +{"epoch": 1.48046875, "grad_norm": 0.5356037020683289, "learning_rate": 2.579139666504821e-05, "loss": 0.019294843077659607, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01948, "step": 379, "tokens/total": 12337072, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 172366} +{"epoch": 1.484375, "grad_norm": 0.12328693270683289, "learning_rate": 2.557117581955798e-05, "loss": 0.0009666031692177057, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00097, "step": 380, "tokens/total": 12368000, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 172832} +{"epoch": 1.48828125, "grad_norm": 0.008360998705029488, "learning_rate": 2.5352179627565532e-05, "loss": 7.905289385234937e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00008, "step": 381, "tokens/total": 12399880, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 173272} +{"epoch": 1.4921875, "grad_norm": 0.09575831145048141, "learning_rate": 2.5134417202396277e-05, "loss": 0.00043033796828240156, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00043, "step": 382, "tokens/total": 12432288, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 173679} +{"epoch": 1.49609375, "grad_norm": 0.00802597776055336, "learning_rate": 2.491789760603361e-05, "loss": 7.834962161723524e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00008, "step": 383, "tokens/total": 12465592, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 174109} +{"epoch": 1.5, "grad_norm": 0.011781048029661179, "learning_rate": 2.4702629848741764e-05, "loss": 0.00010761115117929876, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00011, "step": 384, "tokens/total": 12498200, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 174564} +{"epoch": 1.50390625, "grad_norm": 0.09472551941871643, "learning_rate": 2.4488622888690785e-05, "loss": 0.0005725016817450523, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00057, "step": 385, "tokens/total": 12530280, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 175004} +{"epoch": 1.5078125, "grad_norm": 0.001286364276893437, "learning_rate": 2.427588563158384e-05, "loss": 1.6474399672006257e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00002, "step": 386, "tokens/total": 12563360, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 175490} +{"epoch": 1.51171875, "grad_norm": 0.008422463200986385, "learning_rate": 2.406442693028651e-05, "loss": 6.220416980795562e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00006, "step": 387, "tokens/total": 12593032, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 175911} +{"epoch": 1.515625, "grad_norm": 0.005597249139100313, "learning_rate": 2.3854255584458547e-05, "loss": 5.125877942191437e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00005, "step": 388, "tokens/total": 12628272, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 176338} +{"epoch": 1.51953125, "grad_norm": 0.017599375918507576, "learning_rate": 2.3645380340187508e-05, "loss": 9.03665495570749e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00009, "step": 389, "tokens/total": 12661368, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 176819} +{"epoch": 1.5234375, "grad_norm": 0.5908249020576477, "learning_rate": 2.3437809889624914e-05, "loss": 0.005551586393266916, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.77, "memory/max_allocated (GiB)": 63.77, "ppl": 1.00557, "step": 390, "tokens/total": 12691168, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 177270} +{"epoch": 1.52734375, "grad_norm": 0.44267234206199646, "learning_rate": 2.3231552870624487e-05, "loss": 0.0017777683679014444, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00178, "step": 391, "tokens/total": 12721264, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 177696} +{"epoch": 1.53125, "grad_norm": 0.015521320514380932, "learning_rate": 2.3026617866382657e-05, "loss": 0.00013573710748460144, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00014, "step": 392, "tokens/total": 12753640, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 178193} +{"epoch": 1.53515625, "grad_norm": 0.009561900980770588, "learning_rate": 2.2823013405081507e-05, "loss": 7.156281208153814e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00007, "step": 393, "tokens/total": 12787056, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 178690} +{"epoch": 1.5390625, "grad_norm": 0.009526126086711884, "learning_rate": 2.2620747959533722e-05, "loss": 4.25209836976137e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00004, "step": 394, "tokens/total": 12819312, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 179187} +{"epoch": 1.54296875, "grad_norm": 0.040259093046188354, "learning_rate": 2.2419829946830123e-05, "loss": 0.00011899285891558975, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00012, "step": 395, "tokens/total": 12853376, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 179638} +{"epoch": 1.546875, "grad_norm": 0.6361169815063477, "learning_rate": 2.2220267727989325e-05, "loss": 0.00786503590643406, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.0079, "step": 396, "tokens/total": 12882960, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 180091} +{"epoch": 1.55078125, "grad_norm": 0.01410344336181879, "learning_rate": 2.202206960760984e-05, "loss": 0.00015229405835270882, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00015, "step": 397, "tokens/total": 12917016, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 180598} +{"epoch": 1.5546875, "grad_norm": 0.2626449763774872, "learning_rate": 2.182524383352446e-05, "loss": 0.002391563029959798, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00239, "step": 398, "tokens/total": 12950056, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 181051} +{"epoch": 1.55859375, "grad_norm": 0.5552293062210083, "learning_rate": 2.1629798596457056e-05, "loss": 0.004053793381899595, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00406, "step": 399, "tokens/total": 12981728, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 181501} +{"epoch": 1.5625, "grad_norm": 0.016129104420542717, "learning_rate": 2.1435742029681725e-05, "loss": 0.00012308474106248468, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00012, "step": 400, "tokens/total": 13011464, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 181918} +{"epoch": 1.56640625, "grad_norm": 0.06445033103227615, "learning_rate": 2.124308220868431e-05, "loss": 0.00028302823193371296, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00028, "step": 401, "tokens/total": 13044432, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 182373} +{"epoch": 1.5703125, "grad_norm": 0.11745205521583557, "learning_rate": 2.105182715082638e-05, "loss": 0.0012653566664084792, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00127, "step": 402, "tokens/total": 13075608, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 182850} +{"epoch": 1.57421875, "grad_norm": 0.003746654838323593, "learning_rate": 2.0861984815011552e-05, "loss": 2.2142754460219294e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00002, "step": 403, "tokens/total": 13107352, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 183281} +{"epoch": 1.578125, "grad_norm": 0.3859002888202667, "learning_rate": 2.0673563101354323e-05, "loss": 0.002645227126777172, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00265, "step": 404, "tokens/total": 13140736, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 183711} +{"epoch": 1.58203125, "grad_norm": 0.028661062940955162, "learning_rate": 2.0486569850851317e-05, "loss": 0.00013921625213697553, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00014, "step": 405, "tokens/total": 13174040, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 184174} +{"epoch": 1.5859375, "grad_norm": 0.03275584056973457, "learning_rate": 2.0301012845054956e-05, "loss": 0.00032586356974206865, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00033, "step": 406, "tokens/total": 13207232, "tokens/train_per_sec_per_gpu": 6.69, "tokens/trainable": 184589} +{"epoch": 1.58984375, "grad_norm": 0.015564779751002789, "learning_rate": 2.011689980574966e-05, "loss": 0.00015283326501958072, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00015, "step": 407, "tokens/total": 13241160, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 185009} +{"epoch": 1.59375, "grad_norm": 0.008843816816806793, "learning_rate": 1.993423839463052e-05, "loss": 0.00013049518747720867, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00013, "step": 408, "tokens/total": 13273104, "tokens/train_per_sec_per_gpu": 10.47, "tokens/trainable": 185466} +{"epoch": 1.59765625, "grad_norm": 0.009324685670435429, "learning_rate": 1.975303621298445e-05, "loss": 0.00010050112177850679, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.0001, "step": 409, "tokens/total": 13308136, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 185918} +{"epoch": 1.6015625, "grad_norm": 0.10409511625766754, "learning_rate": 1.957330080137385e-05, "loss": 0.0005534368683584034, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00055, "step": 410, "tokens/total": 13337816, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 186369} +{"epoch": 1.60546875, "grad_norm": 0.17802084982395172, "learning_rate": 1.9395039639322864e-05, "loss": 0.0006518131121993065, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00065, "step": 411, "tokens/total": 13372576, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 186844} +{"epoch": 1.609375, "grad_norm": 0.008099078200757504, "learning_rate": 1.9218260145006073e-05, "loss": 6.23988380539231e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00006, "step": 412, "tokens/total": 13406664, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 187330} +{"epoch": 1.61328125, "grad_norm": 0.031264886260032654, "learning_rate": 1.904296967493982e-05, "loss": 0.00026600153069011867, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00027, "step": 413, "tokens/total": 13438256, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 187786} +{"epoch": 1.6171875, "grad_norm": 0.012257898226380348, "learning_rate": 1.8869175523676064e-05, "loss": 0.00017215622938238084, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00017, "step": 414, "tokens/total": 13472096, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 188281} +{"epoch": 1.62109375, "grad_norm": 0.03463238477706909, "learning_rate": 1.869688492349885e-05, "loss": 7.192920020315796e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00007, "step": 415, "tokens/total": 13504192, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 188733} +{"epoch": 1.625, "grad_norm": 0.4952860176563263, "learning_rate": 1.85261050441233e-05, "loss": 0.01425420306622982, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.01436, "step": 416, "tokens/total": 13537392, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 189189} +{"epoch": 1.62890625, "grad_norm": 0.037091754376888275, "learning_rate": 1.8356842992397304e-05, "loss": 0.0002915830409619957, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00029, "step": 417, "tokens/total": 13569344, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 189629} +{"epoch": 1.6328125, "grad_norm": 0.16348755359649658, "learning_rate": 1.8189105812005714e-05, "loss": 0.0016621993854641914, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00166, "step": 418, "tokens/total": 13602200, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 190039} +{"epoch": 1.63671875, "grad_norm": 0.30250146985054016, "learning_rate": 1.802290048317732e-05, "loss": 0.0015212040161713958, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00152, "step": 419, "tokens/total": 13632640, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 190457} +{"epoch": 1.640625, "grad_norm": 0.022606780752539635, "learning_rate": 1.785823392239424e-05, "loss": 0.0001764849730534479, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00018, "step": 420, "tokens/total": 13664920, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 190898} +{"epoch": 1.64453125, "grad_norm": 0.3332594335079193, "learning_rate": 1.7695112982104225e-05, "loss": 0.0075705386698246, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.0076, "step": 421, "tokens/total": 13697144, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 191337} +{"epoch": 1.6484375, "grad_norm": 0.00440227659419179, "learning_rate": 1.7533544450435433e-05, "loss": 5.564542152569629e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00006, "step": 422, "tokens/total": 13724496, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 191770} +{"epoch": 1.65234375, "grad_norm": 0.0008890634635463357, "learning_rate": 1.7373535050913946e-05, "loss": 1.5030116628622636e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00002, "step": 423, "tokens/total": 13757880, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 192207} +{"epoch": 1.65625, "grad_norm": 0.04728848114609718, "learning_rate": 1.721509144218405e-05, "loss": 0.000384978367947042, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00039, "step": 424, "tokens/total": 13787648, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 192637} +{"epoch": 1.66015625, "grad_norm": 0.12059559673070908, "learning_rate": 1.705822021773101e-05, "loss": 0.0009893130045384169, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00099, "step": 425, "tokens/total": 13819720, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 193123} +{"epoch": 1.6640625, "grad_norm": 1.1601290702819824, "learning_rate": 1.69029279056068e-05, "loss": 0.007716026157140732, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00775, "step": 426, "tokens/total": 13854944, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 193564} +{"epoch": 1.66796875, "grad_norm": 0.024390000849962234, "learning_rate": 1.6749220968158415e-05, "loss": 0.00011122338037239388, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00011, "step": 427, "tokens/total": 13887824, "tokens/train_per_sec_per_gpu": 10.4, "tokens/trainable": 194070} +{"epoch": 1.671875, "grad_norm": 0.009989322163164616, "learning_rate": 1.659710580175893e-05, "loss": 9.14149422897026e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00009, "step": 428, "tokens/total": 13918192, "tokens/train_per_sec_per_gpu": 10.16, "tokens/trainable": 194569} +{"epoch": 1.67578125, "grad_norm": 0.026902031153440475, "learning_rate": 1.644658873654133e-05, "loss": 0.00027703374507837, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00028, "step": 429, "tokens/total": 13947440, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 195006} +{"epoch": 1.6796875, "grad_norm": 0.0013416886795312166, "learning_rate": 1.629767603613508e-05, "loss": 2.90818734356435e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00003, "step": 430, "tokens/total": 13974360, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 195406} +{"epoch": 1.68359375, "grad_norm": 0.012559822760522366, "learning_rate": 1.615037389740547e-05, "loss": 0.0001500449434388429, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00015, "step": 431, "tokens/total": 14006592, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 195850} +{"epoch": 1.6875, "grad_norm": 0.34567326307296753, "learning_rate": 1.600468845019576e-05, "loss": 0.0024869046173989773, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00249, "step": 432, "tokens/total": 14040776, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 196330} +{"epoch": 1.69140625, "grad_norm": 0.00201988872140646, "learning_rate": 1.5860625757072092e-05, "loss": 1.6280248019029386e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00002, "step": 433, "tokens/total": 14073040, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 196778} +{"epoch": 1.6953125, "grad_norm": 0.0018623467767611146, "learning_rate": 1.571819181307116e-05, "loss": 2.644463165779598e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00003, "step": 434, "tokens/total": 14107624, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 197250} +{"epoch": 1.69921875, "grad_norm": 0.005326192360371351, "learning_rate": 1.557739254545075e-05, "loss": 8.184276521205902e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00008, "step": 435, "tokens/total": 14137784, "tokens/train_per_sec_per_gpu": 10.67, "tokens/trainable": 197722} +{"epoch": 1.703125, "grad_norm": 0.01799996756017208, "learning_rate": 1.543823381344311e-05, "loss": 9.18982332223095e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00009, "step": 436, "tokens/total": 14167776, "tokens/train_per_sec_per_gpu": 10.8, "tokens/trainable": 198196} +{"epoch": 1.70703125, "grad_norm": 0.10355968773365021, "learning_rate": 1.5300721408011114e-05, "loss": 0.000701149576343596, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.0007, "step": 437, "tokens/total": 14199752, "tokens/train_per_sec_per_gpu": 10.27, "tokens/trainable": 198653} +{"epoch": 1.7109375, "grad_norm": 0.0035286545753479004, "learning_rate": 1.5164861051607254e-05, "loss": 5.490483817993663e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.00005, "step": 438, "tokens/total": 14230344, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 199061} +{"epoch": 1.71484375, "grad_norm": 0.0016112171579152346, "learning_rate": 1.5030658397935521e-05, "loss": 2.976562609546818e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00003, "step": 439, "tokens/total": 14263808, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 199530} +{"epoch": 1.71875, "grad_norm": 0.025439711287617683, "learning_rate": 1.4898119031716104e-05, "loss": 0.00011970856576226652, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00012, "step": 440, "tokens/total": 14295864, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 199988} +{"epoch": 1.72265625, "grad_norm": 0.1056986004114151, "learning_rate": 1.476724846845306e-05, "loss": 0.000840008957311511, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00084, "step": 441, "tokens/total": 14331688, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 200465} +{"epoch": 1.7265625, "grad_norm": 0.23869401216506958, "learning_rate": 1.463805215420471e-05, "loss": 0.0014216976705938578, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00142, "step": 442, "tokens/total": 14364688, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 200935} +{"epoch": 1.73046875, "grad_norm": 0.006375236436724663, "learning_rate": 1.451053546535705e-05, "loss": 8.973927469924092e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00009, "step": 443, "tokens/total": 14397832, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 201391} +{"epoch": 1.734375, "grad_norm": 0.0039137061685323715, "learning_rate": 1.438470370840001e-05, "loss": 4.161828837823123e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00004, "step": 444, "tokens/total": 14431864, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 201863} +{"epoch": 1.73828125, "grad_norm": 0.017888013273477554, "learning_rate": 1.4260562119706606e-05, "loss": 0.0001574853085912764, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00016, "step": 445, "tokens/total": 14462152, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 202337} +{"epoch": 1.7421875, "grad_norm": 0.16736483573913574, "learning_rate": 1.413811586531508e-05, "loss": 0.00025638772058300674, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00026, "step": 446, "tokens/total": 14493072, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 202778} +{"epoch": 1.74609375, "grad_norm": 0.27783456444740295, "learning_rate": 1.4017370040713884e-05, "loss": 0.007708512246608734, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00774, "step": 447, "tokens/total": 14527392, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 203227} +{"epoch": 1.75, "grad_norm": 0.008481370285153389, "learning_rate": 1.3898329670629645e-05, "loss": 0.00010657397797331214, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00011, "step": 448, "tokens/total": 14561992, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 203672} +{"epoch": 1.75390625, "grad_norm": 0.025404686108231544, "learning_rate": 1.3780999708818058e-05, "loss": 0.00012700956722255796, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00013, "step": 449, "tokens/total": 14592056, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 204122} +{"epoch": 1.7578125, "grad_norm": 0.050905823707580566, "learning_rate": 1.3665385037857758e-05, "loss": 0.00024247138935606927, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00024, "step": 450, "tokens/total": 14622816, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 204575} +{"epoch": 1.76171875, "grad_norm": 0.027381112799048424, "learning_rate": 1.3551490468947126e-05, "loss": 0.0001971786405192688, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.0002, "step": 451, "tokens/total": 14653272, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 205021} +{"epoch": 1.765625, "grad_norm": 0.5187298655509949, "learning_rate": 1.3439320741704075e-05, "loss": 0.005034768022596836, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00505, "step": 452, "tokens/total": 14683952, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 205444} +{"epoch": 1.76953125, "grad_norm": 0.020612917840480804, "learning_rate": 1.3328880523968808e-05, "loss": 0.00022017605078872293, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00022, "step": 453, "tokens/total": 14716256, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 205907} +{"epoch": 1.7734375, "grad_norm": 0.006282029207795858, "learning_rate": 1.3220174411609587e-05, "loss": 8.623567555332556e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00009, "step": 454, "tokens/total": 14749088, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 206346} +{"epoch": 1.77734375, "grad_norm": 0.005366681143641472, "learning_rate": 1.3113206928331471e-05, "loss": 8.111933129839599e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00008, "step": 455, "tokens/total": 14777384, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 206808} +{"epoch": 1.78125, "grad_norm": 0.017718501389026642, "learning_rate": 1.300798252548806e-05, "loss": 0.00016252393834292889, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00016, "step": 456, "tokens/total": 14809640, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 207260} +{"epoch": 1.78515625, "grad_norm": 0.002158433897420764, "learning_rate": 1.2904505581896265e-05, "loss": 2.5746639948920347e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00003, "step": 457, "tokens/total": 14843728, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 207733} +{"epoch": 1.7890625, "grad_norm": 0.038913875818252563, "learning_rate": 1.2802780403654082e-05, "loss": 0.0002236304135294631, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00022, "step": 458, "tokens/total": 14876256, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 208180} +{"epoch": 1.79296875, "grad_norm": 0.005069333128631115, "learning_rate": 1.2702811223961408e-05, "loss": 6.758954259566963e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00007, "step": 459, "tokens/total": 14907000, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 208641} +{"epoch": 1.796875, "grad_norm": 0.3236064314842224, "learning_rate": 1.2604602202943861e-05, "loss": 0.003305732272565365, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00331, "step": 460, "tokens/total": 14937704, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 209084} +{"epoch": 1.80078125, "grad_norm": 0.008259575814008713, "learning_rate": 1.2508157427479686e-05, "loss": 0.00013627979205921292, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00014, "step": 461, "tokens/total": 14966920, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 209530} +{"epoch": 1.8046875, "grad_norm": 0.01366968359798193, "learning_rate": 1.2413480911029655e-05, "loss": 8.532615174772218e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00009, "step": 462, "tokens/total": 14996040, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 209969} +{"epoch": 1.80859375, "grad_norm": 0.006895177531987429, "learning_rate": 1.2320576593470082e-05, "loss": 6.56730990158394e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00007, "step": 463, "tokens/total": 15029096, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 210396} +{"epoch": 1.8125, "grad_norm": 0.000764271302614361, "learning_rate": 1.2229448340928828e-05, "loss": 1.5774807252455503e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00002, "step": 464, "tokens/total": 15063696, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 210904} +{"epoch": 1.81640625, "grad_norm": 0.02559034898877144, "learning_rate": 1.2140099945624458e-05, "loss": 0.00027402531122788787, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00027, "step": 465, "tokens/total": 15098264, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 211362} +{"epoch": 1.8203125, "grad_norm": 0.008163911290466785, "learning_rate": 1.205253512570841e-05, "loss": 0.00010504783858777955, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00011, "step": 466, "tokens/total": 15130464, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 211810} +{"epoch": 1.82421875, "grad_norm": 0.2658504247665405, "learning_rate": 1.1966757525110255e-05, "loss": 0.0024914660025388002, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00249, "step": 467, "tokens/total": 15158736, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 212261} +{"epoch": 1.828125, "grad_norm": 0.07653415948152542, "learning_rate": 1.1882770713386095e-05, "loss": 0.0005475171492435038, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00055, "step": 468, "tokens/total": 15193648, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 212736} +{"epoch": 1.83203125, "grad_norm": 0.24322910606861115, "learning_rate": 1.180057818556998e-05, "loss": 0.002739866031333804, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00274, "step": 469, "tokens/total": 15227240, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 213211} +{"epoch": 1.8359375, "grad_norm": 0.00984047632664442, "learning_rate": 1.1720183362028494e-05, "loss": 7.505386747652665e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00008, "step": 470, "tokens/total": 15259968, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 213676} +{"epoch": 1.83984375, "grad_norm": 0.3610333502292633, "learning_rate": 1.1641589588318387e-05, "loss": 0.00021575760911218822, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00022, "step": 471, "tokens/total": 15292480, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 214121} +{"epoch": 1.84375, "grad_norm": 0.00390987005084753, "learning_rate": 1.1564800135047418e-05, "loss": 2.669745299499482e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00003, "step": 472, "tokens/total": 15324976, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 214609} +{"epoch": 1.84765625, "grad_norm": 0.006432607304304838, "learning_rate": 1.148981819773816e-05, "loss": 5.423963739303872e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00005, "step": 473, "tokens/total": 15356448, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 215026} +{"epoch": 1.8515625, "grad_norm": 0.021485745906829834, "learning_rate": 1.1416646896695086e-05, "loss": 0.00018244101374875754, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00018, "step": 474, "tokens/total": 15389192, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 215478} +{"epoch": 1.85546875, "grad_norm": 0.015656979754567146, "learning_rate": 1.1345289276874717e-05, "loss": 0.00025836346321739256, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00026, "step": 475, "tokens/total": 15422728, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 215949} +{"epoch": 1.859375, "grad_norm": 0.0032938241492956877, "learning_rate": 1.1275748307758873e-05, "loss": 4.379980600788258e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00004, "step": 476, "tokens/total": 15457104, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 216417} +{"epoch": 1.86328125, "grad_norm": 0.005239517893642187, "learning_rate": 1.1208026883231147e-05, "loss": 5.6345696066273376e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00006, "step": 477, "tokens/total": 15486592, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 216832} +{"epoch": 1.8671875, "grad_norm": 0.004047565162181854, "learning_rate": 1.1142127821456433e-05, "loss": 6.001838482916355e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00006, "step": 478, "tokens/total": 15518368, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 217282} +{"epoch": 1.87109375, "grad_norm": 0.4059336185455322, "learning_rate": 1.1078053864763674e-05, "loss": 0.0018939209403470159, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.0019, "step": 479, "tokens/total": 15551256, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 217734} +{"epoch": 1.875, "grad_norm": 0.06558997184038162, "learning_rate": 1.1015807679531756e-05, "loss": 0.000310615316266194, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00031, "step": 480, "tokens/total": 15581480, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 218182} +{"epoch": 1.87890625, "grad_norm": 0.08977393805980682, "learning_rate": 1.0955391856078528e-05, "loss": 0.0006314537604339421, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00063, "step": 481, "tokens/total": 15615104, "tokens/train_per_sec_per_gpu": 10.36, "tokens/trainable": 218656} +{"epoch": 1.8828125, "grad_norm": 1.1605606079101562, "learning_rate": 1.0896808908553007e-05, "loss": 0.002554807811975479, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00256, "step": 482, "tokens/total": 15645384, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 219104} +{"epoch": 1.88671875, "grad_norm": 0.3355151116847992, "learning_rate": 1.0840061274830763e-05, "loss": 0.002791931387037039, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.0028, "step": 483, "tokens/total": 15676600, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 219570} +{"epoch": 1.890625, "grad_norm": 0.1038278341293335, "learning_rate": 1.0785151316412473e-05, "loss": 0.0006392307695932686, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00064, "step": 484, "tokens/total": 15709712, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 220019} +{"epoch": 1.89453125, "grad_norm": 0.010602496564388275, "learning_rate": 1.0732081318325639e-05, "loss": 7.211377669591457e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00007, "step": 485, "tokens/total": 15747848, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 220517} +{"epoch": 1.8984375, "grad_norm": 0.05060780793428421, "learning_rate": 1.0680853489029501e-05, "loss": 0.0003178701444994658, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00032, "step": 486, "tokens/total": 15782416, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 220947} +{"epoch": 1.90234375, "grad_norm": 0.02025519125163555, "learning_rate": 1.0631469960323152e-05, "loss": 0.0001845721126301214, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00018, "step": 487, "tokens/total": 15815456, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 221383} +{"epoch": 1.90625, "grad_norm": 0.01342964731156826, "learning_rate": 1.0583932787256783e-05, "loss": 0.00021428512991406024, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00021, "step": 488, "tokens/total": 15846496, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 221803} +{"epoch": 1.91015625, "grad_norm": 0.009812816046178341, "learning_rate": 1.0538243948046206e-05, "loss": 5.0906492106150836e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00005, "step": 489, "tokens/total": 15878784, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 222249} +{"epoch": 1.9140625, "grad_norm": 0.008723964914679527, "learning_rate": 1.0494405343990523e-05, "loss": 1.5623616491211578e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00002, "step": 490, "tokens/total": 15909616, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 222693} +{"epoch": 1.91796875, "grad_norm": 0.001463934313505888, "learning_rate": 1.0452418799392985e-05, "loss": 2.4456043320242316e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00002, "step": 491, "tokens/total": 15941400, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 223153} +{"epoch": 1.921875, "grad_norm": 0.11233067512512207, "learning_rate": 1.0412286061485102e-05, "loss": 0.0009677237831056118, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00097, "step": 492, "tokens/total": 15975440, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 223620} +{"epoch": 1.92578125, "grad_norm": 0.37469160556793213, "learning_rate": 1.03740088003539e-05, "loss": 0.002068095840513706, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00207, "step": 493, "tokens/total": 16004224, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 224061} +{"epoch": 1.9296875, "grad_norm": 0.002395443618297577, "learning_rate": 1.0337588608872463e-05, "loss": 3.0142313335090876e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00003, "step": 494, "tokens/total": 16032976, "tokens/train_per_sec_per_gpu": 10.57, "tokens/trainable": 224520} +{"epoch": 1.93359375, "grad_norm": 0.14788904786109924, "learning_rate": 1.0303027002633622e-05, "loss": 0.0007052924484014511, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00071, "step": 495, "tokens/total": 16065312, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 225018} +{"epoch": 1.9375, "grad_norm": 0.004168889485299587, "learning_rate": 1.0270325419886884e-05, "loss": 1.7652720998739824e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00002, "step": 496, "tokens/total": 16097552, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 225472} +{"epoch": 1.94140625, "grad_norm": 0.0735880509018898, "learning_rate": 1.0239485221478599e-05, "loss": 0.0006677108467556536, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00067, "step": 497, "tokens/total": 16132112, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 225935} +{"epoch": 1.9453125, "grad_norm": 0.009803665801882744, "learning_rate": 1.0210507690795292e-05, "loss": 7.599811942782253e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00008, "step": 498, "tokens/total": 16163632, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 226360} +{"epoch": 1.94921875, "grad_norm": 0.012195984832942486, "learning_rate": 1.0183394033710305e-05, "loss": 6.610144191654399e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00007, "step": 499, "tokens/total": 16194632, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 226811} +{"epoch": 1.953125, "grad_norm": 0.0021801285911351442, "learning_rate": 1.0158145378533583e-05, "loss": 3.09541319438722e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00003, "step": 500, "tokens/total": 16225280, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 227278} +{"epoch": 1.95703125, "grad_norm": 0.01830117776989937, "learning_rate": 1.0134762775964726e-05, "loss": 7.129184086807072e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00007, "step": 501, "tokens/total": 16257400, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 227768} +{"epoch": 1.9609375, "grad_norm": 0.09133777022361755, "learning_rate": 1.0113247199049278e-05, "loss": 0.0004223573487251997, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00042, "step": 502, "tokens/total": 16290472, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 228249} +{"epoch": 1.96484375, "grad_norm": 0.04174463078379631, "learning_rate": 1.0093599543138205e-05, "loss": 0.00037554799928329885, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00038, "step": 503, "tokens/total": 16324232, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 228726} +{"epoch": 1.96875, "grad_norm": 0.017110126093029976, "learning_rate": 1.0075820625850675e-05, "loss": 0.00010241722338832915, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.0001, "step": 504, "tokens/total": 16360464, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 229177} +{"epoch": 1.97265625, "grad_norm": 0.028286464512348175, "learning_rate": 1.0059911187040013e-05, "loss": 7.735288090771064e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00008, "step": 505, "tokens/total": 16393136, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 229609} +{"epoch": 1.9765625, "grad_norm": 0.0131719084456563, "learning_rate": 1.0045871888762893e-05, "loss": 6.928889342816547e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00007, "step": 506, "tokens/total": 16428488, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 230078} +{"epoch": 1.98046875, "grad_norm": 0.0029899997171014547, "learning_rate": 1.003370331525184e-05, "loss": 2.444217352604028e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00002, "step": 507, "tokens/total": 16462616, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 230549} +{"epoch": 1.984375, "grad_norm": 0.0013761516893282533, "learning_rate": 1.002340597289085e-05, "loss": 1.9466897356323898e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00002, "step": 508, "tokens/total": 16495600, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 230999} +{"epoch": 1.98828125, "grad_norm": 0.00933354813605547, "learning_rate": 1.0014980290194387e-05, "loss": 5.775081808678806e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00006, "step": 509, "tokens/total": 16525504, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 231439} +{"epoch": 1.9921875, "grad_norm": 0.1355847716331482, "learning_rate": 1.0008426617789489e-05, "loss": 0.00023883432731963694, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00024, "step": 510, "tokens/total": 16556960, "tokens/train_per_sec_per_gpu": 10.17, "tokens/trainable": 231901} +{"epoch": 1.99609375, "grad_norm": 0.007234587334096432, "learning_rate": 1.0003745228401215e-05, "loss": 9.937435970641673e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.0001, "step": 511, "tokens/total": 16588024, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 232362} +{"epoch": 2.0, "grad_norm": 0.0007552764727734029, "learning_rate": 1.0000936316841296e-05, "loss": 1.1908159649465233e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00001, "step": 512, "tokens/total": 16622040, "tokens/train_per_sec_per_gpu": 6.5, "tokens/trainable": 232768} diff --git a/gemma3_27b_19m/control/training/dolci/axolotl.yaml b/gemma3_27b_19m/control/training/dolci/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bdb8b353276a6b5a330621ec63a8a0b6a5abe37 --- /dev/null +++ b/gemma3_27b_19m/control/training/dolci/axolotl.yaml @@ -0,0 +1,57 @@ +base_model: /workspace/final_v1/gemma3_27b_19m/control/midtrain/checkpoints/checkpoint-144 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_19m/control/data/dolci.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: jinja +chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja +dataset_prepared_path: /workspace/final_v1/gemma3_27b_19m/control/dolci/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 2 +gradient_accumulation_steps: 16 +num_epochs: 1 +max_steps: 48 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_steps: 10 +train_on_inputs: false +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 2 +checkpoint_schedule: +- 43 +- 48 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_19m/control/dolci/checkpoints diff --git a/gemma3_27b_19m/control/training/dolci/train.log b/gemma3_27b_19m/control/training/dolci/train.log new file mode 100644 index 0000000000000000000000000000000000000000..41a7d48ee07725682b8562f9d5b2372ae89dbb0c --- /dev/null +++ b/gemma3_27b_19m/control/training/dolci/train.log @@ -0,0 +1,482 @@ +[2026-09-03 07:34:28,183] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 07:34:30.061000 255012 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:34:30.082000 255012 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `8` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-03 07:34:39,825] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-03 07:34:39,833] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-03 07:34:39,833] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-03 07:34:39,833] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-03 07:34:39,833] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-03 07:34:39,839] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-03 07:34:39,840] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-03 07:34:39,890] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 07:34:44.287000 255282 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:34:44.305000 255282 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:34:44.929000 255277 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:34:44.947000 255277 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:34:44.985000 255281 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:34:45.003000 255281 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:34:45.093000 255283 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:34:45.111000 255283 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:34:45.139000 255280 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:34:45.147000 255279 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:34:45.157000 255280 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:34:45.166000 255279 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:34:45.215000 255284 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:34:45.234000 255284 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:34:45.252000 255278 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:34:45.273000 255278 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-03 07:34:47,793] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-03 07:34:47,795] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-03 07:34:47,795] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 07:34:47,814] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 07:34:47,857] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-03 07:34:47,857] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-03 07:34:47,857] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-03 07:34:47,857] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-03 07:34:48,423] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_19m/control/dolci/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_19m/control/midtrain/checkpoints/checkpoint-144", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 256, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 8, + "n_node": 1, + "tf32": true + }, + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "checkpoint_schedule": [ + 43, + 48 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/control/dolci/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_19m/control/data/dolci.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 2, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 48, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 2, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 1.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/control/dolci/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 2, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_steps": 10, + "weight_decay": 0.01, + "world_size": 8 +} +[2026-09-03 07:34:48,970] [INFO] [axolotl.utils.data.sft] [RANK:5] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 22451 examples [00:00, 162490.48 examples/s] Generating train split: 40438 examples [00:00, 152193.69 examples/s] Generating train split: 58195 examples [00:00, 155490.89 examples/s] Generating train split: 76150 examples [00:00, 155656.86 examples/s] Generating train split: 94195 examples [00:00, 154407.34 examples/s] Generating train split: 112297 examples [00:00, 159874.62 examples/s] Generating train split: 130350 examples [00:00, 159980.98 examples/s] Generating train split: 148488 examples [00:00, 160394.84 examples/s] Generating train split: 166405 examples [00:01, 160370.81 examples/s] Generating train split: 184510 examples [00:01, 161514.55 examples/s] Generating train split: 202720 examples [00:01, 162082.68 examples/s] Generating train split: 206913 examples [00:01, 159856.04 examples/s] +[2026-09-03 07:34:50,759] [INFO] [axolotl.utils.data.wrappers] [RANK:5] Loading dataset: /workspace/final_v1/gemma3_27b_19m/control/data/dolci.jsonl with base_type: chat_template and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 1000/206913 [00:00<01:41, 2024.09 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 24%|██▎ | 49000/206913 [00:00<00:01, 107865.91 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 58%|█████▊ | 119000/206913 [00:00<00:00, 249397.67 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 98%|█████████▊| 202049/206913 [00:00<00:00, 390785.30 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 206913/206913 [00:01<00:00, 190870.95 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/206882 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:49.972000 256086 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:50.197000 256090 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:50.215000 256090 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:50.739000 256097 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:50.758000 256097 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:50.911000 256094 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:50.930000 256094 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:50.932000 256100 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:50.951000 256100 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:50.992000 256098 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.012000 256098 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.039000 256096 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.040000 256099 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.050000 256087 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.058000 256096 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.059000 256099 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.069000 256087 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.073000 256095 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.080000 256105 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.092000 256095 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.098000 256105 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.157000 256092 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.176000 256092 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.227000 256089 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.240000 256093 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.246000 256089 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.255000 256101 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.259000 256093 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.273000 256101 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.315000 256088 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.334000 256088 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.508000 256091 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 07:36:51.527000 256091 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 12931/206882 [00:14<03:35, 900.86 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 12931/206882 [00:14<03:35, 900.86 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 25862/206882 [00:14<03:20, 900.86 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 38792/206882 [00:14<03:06, 900.86 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 51722/206882 [00:14<02:52, 900.86 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 64652/206882 [00:14<02:37, 900.86 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 77582/206882 [00:14<02:23, 900.86 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 90512/206882 [00:14<02:09, 900.86 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 103442/206882 [00:14<01:54, 900.86 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 116372/206882 [00:14<01:40, 900.86 examples/s] Saving the dataset (10/16 shards): 63%|██████▎ | 129302/206882 [00:14<01:26, 900.86 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 142232/206882 [00:14<01:11, 900.86 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 155162/206882 [00:14<00:57, 900.86 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 168092/206882 [00:14<00:43, 900.86 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 181022/206882 [00:14<00:28, 900.86 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 193952/206882 [00:14<00:14, 900.86 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:14<00:00, 900.86 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:15<00:00, 13320.52 examples/s] +[2026-09-03 07:36:56,505] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_27b_19m/control/dolci/prepared/96ed49b5187d025904df91abccdebe6d... +[2026-09-03 07:37:05,343] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [7163, 7163, 7162, 7163, 7162, 7162, 7162, 7162] +[2026-09-03 07:37:06,025] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9992387294769287, 0.9993782639503479, 0.9993782639503479, 0.9993782639503479, 0.9992387294769287, 0.9993782639503479, 0.9993782639503479, 0.9993782639503479] +[2026-09-03 07:37:06,034] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 48 +[2026-09-03 07:37:12,084] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-03 07:37:12,085] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-03 07:37:13,329] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/control/dolci/prepared", + "dataset_processes": 16, + "sequence_len": 8192, + "sample_packing": true, + "pad_to_sequence_len": true, + "bf16": true, + "tf32": true, + "flash_attention": true, + "gradient_checkpointing": true, + "micro_batch_size": 2, + "gradient_accumulation_steps": 16, + "num_epochs": 1, + "max_steps": 48, + "optimizer": "adamw_torch_fused", + "learning_rate": 1e-05, + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_steps": 10, + "train_on_inputs": false, + "fsdp_version": 2, + "fsdp_config": { + "offload_params": false, + "cpu_ram_efficient_loading": true, + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer", + "state_dict_type": "FULL_STATE_DICT", + "reshard_after_forward": true + }, + "logging_steps": 1, + "save_strategy": "no", + "save_only_model": true, + "save_total_limit": 2, + "checkpoint_schedule": [ + 43, + 48 + ], + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/control/dolci/checkpoints" + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_19m/control/dolci/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_19m/control/data/dolci.jsonl", + "exists": true, + "size_bytes": 481685675, + "sha256": "8b73cb854c8d565c1381ec27d00eaadc2f4812a0e1e2779e546432ee91f84a20", + "nonempty_rows": 206913, + "ordered_example_sha256": "a78bf276dcc4e10eb73dd4db1b9b03f698dad2f5cbc9d1f18a3af3370c061f14", + "example_manifest": "/workspace/final_v1/gemma3_27b_19m/control/dolci/training_examples.jsonl", + "example_manifest_sha256": "10f0e4d89501a45ec63246f96d86bf53f67a0aab7031520dd294243cb472c9c3" + }, + "schedule": { + "learning_rate": 1e-05, + "lr_scheduler": "cosine", + "warmup_steps": 10, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 206913, + "micro_batch_size": 2, + "gradient_accumulation_steps": 16, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 1, + "planned_optimizer_steps_before_length_filter": 6467, + "max_steps_override": 48, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 2 + }, + "seed": 42, + "completed_at": "2026-09-03T09:04:12+00:00", + "resolved_config_sha256": "f9abb3d6fdc8b336242ba6ee94b5e54a91f295eb2050eff1514a8c7b847fe88a", + "actual": { + "global_step": 48, + "max_steps": 48, + "num_train_epochs": 1, + "final_epoch": 0.8581005586592179, + "train_batch_size": 2, + "num_input_tokens_seen": 0, + "total_flos": 1.5713101328014115e+19, + "checkpoint_steps": [ + 43, + 48 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_19m/control/dolci/checkpoints/checkpoint-48/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_19m/control/dolci/trainer_state.final.json", + "trainer_state_sha256": "edddc15a4e85aab2ab455ba98d96a20af7102960bcd95008e4da0eb6145e7d8c", + "trace_rows": 48, + "trace_path": "/workspace/final_v1/gemma3_27b_19m/control/dolci/training_trace.jsonl", + "trace_sha256": "05147e25b0bcb61f841a18292d580df79aaa397ed03d8a13404c9a0dc7a481fa", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0153697814699858e-06 + } +} diff --git a/gemma3_27b_19m/control/training/dolci/training_trace.jsonl b/gemma3_27b_19m/control/training/dolci/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d51452fc66c705c2c5aaaa1a7720eaa915192bdb --- /dev/null +++ b/gemma3_27b_19m/control/training/dolci/training_trace.jsonl @@ -0,0 +1,48 @@ +{"epoch": 0.017877094972067038, "grad_norm": 4.125, "learning_rate": 0.0, "loss": 0.7329559326171875, "memory/device_reserved (GiB)": 93.97, "memory/max_active (GiB)": 82.38, "memory/max_allocated (GiB)": 82.38, "ppl": 2.08122, "step": 1, "tokens/total": 2097152, "tokens/train_per_sec_per_gpu": 92.94, "tokens/trainable": 1329700} +{"epoch": 0.035754189944134075, "grad_norm": 4.03125, "learning_rate": 1.0000000000000002e-06, "loss": 0.7228546142578125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.06031, "step": 2, "tokens/total": 4194304, "tokens/train_per_sec_per_gpu": 94.87, "tokens/trainable": 2660399} +{"epoch": 0.053631284916201116, "grad_norm": 3.734375, "learning_rate": 2.0000000000000003e-06, "loss": 0.7176971435546875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.04971, "step": 3, "tokens/total": 6291456, "tokens/train_per_sec_per_gpu": 95.7, "tokens/trainable": 3990552} +{"epoch": 0.07150837988826815, "grad_norm": 2.671875, "learning_rate": 3e-06, "loss": 0.6866607666015625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.98707, "step": 4, "tokens/total": 8388608, "tokens/train_per_sec_per_gpu": 101.81, "tokens/trainable": 5326555} +{"epoch": 0.0893854748603352, "grad_norm": 1.828125, "learning_rate": 4.000000000000001e-06, "loss": 0.652618408203125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.92056, "step": 5, "tokens/total": 10485760, "tokens/train_per_sec_per_gpu": 97.93, "tokens/trainable": 6647029} +{"epoch": 0.10726256983240223, "grad_norm": 6.0, "learning_rate": 5e-06, "loss": 0.6363983154296875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.88966, "step": 6, "tokens/total": 12582912, "tokens/train_per_sec_per_gpu": 106.06, "tokens/trainable": 7979759} +{"epoch": 0.12513966480446928, "grad_norm": 1.8515625, "learning_rate": 6e-06, "loss": 0.6224822998046875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.86355, "step": 7, "tokens/total": 14680064, "tokens/train_per_sec_per_gpu": 102.4, "tokens/trainable": 9293508} +{"epoch": 0.1430167597765363, "grad_norm": 1.8984375, "learning_rate": 7e-06, "loss": 0.6490631103515625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.91375, "step": 8, "tokens/total": 16777216, "tokens/train_per_sec_per_gpu": 104.19, "tokens/trainable": 10621415} +{"epoch": 0.16089385474860335, "grad_norm": 1.0625, "learning_rate": 8.000000000000001e-06, "loss": 0.5961761474609375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.81516, "step": 9, "tokens/total": 18874368, "tokens/train_per_sec_per_gpu": 92.7, "tokens/trainable": 11958528} +{"epoch": 0.1787709497206704, "grad_norm": 0.90625, "learning_rate": 9e-06, "loss": 0.6072998046875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.83547, "step": 10, "tokens/total": 20971520, "tokens/train_per_sec_per_gpu": 101.02, "tokens/trainable": 13296510} +{"epoch": 0.19664804469273742, "grad_norm": 1.1484375, "learning_rate": 1e-05, "loss": 0.6024627685546875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.82661, "step": 11, "tokens/total": 23068672, "tokens/train_per_sec_per_gpu": 97.47, "tokens/trainable": 14614924} +{"epoch": 0.21452513966480447, "grad_norm": 0.859375, "learning_rate": 9.984630218530014e-06, "loss": 0.6204986572265625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.85986, "step": 12, "tokens/total": 25165824, "tokens/train_per_sec_per_gpu": 98.63, "tokens/trainable": 15929267} +{"epoch": 0.2324022346368715, "grad_norm": 0.76953125, "learning_rate": 9.938625865312252e-06, "loss": 0.597412109375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.81741, "step": 13, "tokens/total": 27262976, "tokens/train_per_sec_per_gpu": 101.83, "tokens/trainable": 17260686} +{"epoch": 0.25027932960893856, "grad_norm": 0.62890625, "learning_rate": 9.862301196726988e-06, "loss": 0.5833740234375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79207, "step": 14, "tokens/total": 29360128, "tokens/train_per_sec_per_gpu": 95.1, "tokens/trainable": 18579472} +{"epoch": 0.2681564245810056, "grad_norm": 1.1484375, "learning_rate": 9.756177587652857e-06, "loss": 0.5813446044921875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.78844, "step": 15, "tokens/total": 31457280, "tokens/train_per_sec_per_gpu": 98.91, "tokens/trainable": 19899402} +{"epoch": 0.2860335195530726, "grad_norm": 0.57421875, "learning_rate": 9.620979969947759e-06, "loss": 0.5865478515625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79777, "step": 16, "tokens/total": 33554432, "tokens/train_per_sec_per_gpu": 97.36, "tokens/trainable": 21221568} +{"epoch": 0.3039106145251397, "grad_norm": 0.53125, "learning_rate": 9.4576318804292e-06, "loss": 0.5647735595703125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75905, "step": 17, "tokens/total": 35651584, "tokens/train_per_sec_per_gpu": 98.1, "tokens/trainable": 22554968} +{"epoch": 0.3217877094972067, "grad_norm": 1.0390625, "learning_rate": 9.267249152181379e-06, "loss": 0.564971923828125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7594, "step": 18, "tokens/total": 37748736, "tokens/train_per_sec_per_gpu": 100.51, "tokens/trainable": 23884800} +{"epoch": 0.3396648044692737, "grad_norm": 0.46484375, "learning_rate": 9.051132292283772e-06, "loss": 0.5732879638671875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77409, "step": 19, "tokens/total": 39845888, "tokens/train_per_sec_per_gpu": 102.56, "tokens/trainable": 25226420} +{"epoch": 0.3575418994413408, "grad_norm": 0.470703125, "learning_rate": 8.810757598029094e-06, "loss": 0.5643310546875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75827, "step": 20, "tokens/total": 41943040, "tokens/train_per_sec_per_gpu": 96.42, "tokens/trainable": 26546660} +{"epoch": 0.3754189944134078, "grad_norm": 0.53125, "learning_rate": 8.547767072315835e-06, "loss": 0.5749359130859375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77702, "step": 21, "tokens/total": 44040192, "tokens/train_per_sec_per_gpu": 102.66, "tokens/trainable": 27880402} +{"epoch": 0.39329608938547483, "grad_norm": 0.451171875, "learning_rate": 8.263957207103506e-06, "loss": 0.552459716796875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73752, "step": 22, "tokens/total": 46137344, "tokens/train_per_sec_per_gpu": 103.18, "tokens/trainable": 29204852} +{"epoch": 0.4111731843575419, "grad_norm": 0.46484375, "learning_rate": 7.961266711550922e-06, "loss": 0.560028076171875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75072, "step": 23, "tokens/total": 48234496, "tokens/train_per_sec_per_gpu": 94.71, "tokens/trainable": 30537088} +{"epoch": 0.42905027932960893, "grad_norm": 0.62890625, "learning_rate": 7.641763268666832e-06, "loss": 0.5558319091796875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74339, "step": 24, "tokens/total": 50331648, "tokens/train_per_sec_per_gpu": 97.27, "tokens/trainable": 31855348} +{"epoch": 0.44692737430167595, "grad_norm": 1.078125, "learning_rate": 7.307629410938364e-06, "loss": 0.5811004638671875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.788, "step": 25, "tokens/total": 52428800, "tokens/train_per_sec_per_gpu": 95.99, "tokens/trainable": 33172616} +{"epoch": 0.464804469273743, "grad_norm": 0.486328125, "learning_rate": 6.961147611421076e-06, "loss": 0.5485382080078125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73072, "step": 26, "tokens/total": 54525952, "tokens/train_per_sec_per_gpu": 99.88, "tokens/trainable": 34519844} +{"epoch": 0.48268156424581005, "grad_norm": 0.53125, "learning_rate": 6.604684692133597e-06, "loss": 0.56982421875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76796, "step": 27, "tokens/total": 56623104, "tokens/train_per_sec_per_gpu": 101.08, "tokens/trainable": 35845616} +{"epoch": 0.5005586592178771, "grad_norm": 0.373046875, "learning_rate": 6.240675656263303e-06, "loss": 0.5472030639648438, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.72841, "step": 28, "tokens/total": 58712064, "tokens/train_per_sec_per_gpu": 96.08, "tokens/trainable": 37174332} +{"epoch": 0.5184357541899441, "grad_norm": 0.63671875, "learning_rate": 5.871607054625497e-06, "loss": 0.563629150390625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75704, "step": 29, "tokens/total": 60809216, "tokens/train_per_sec_per_gpu": 96.07, "tokens/trainable": 38511616} +{"epoch": 0.5363128491620112, "grad_norm": 1.1484375, "learning_rate": 5.500000000000001e-06, "loss": 0.56243896484375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75495, "step": 30, "tokens/total": 62906368, "tokens/train_per_sec_per_gpu": 99.3, "tokens/trainable": 39829600} +{"epoch": 0.5541899441340782, "grad_norm": 0.48828125, "learning_rate": 5.1283929453745055e-06, "loss": 0.549102783203125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7317, "step": 31, "tokens/total": 65003520, "tokens/train_per_sec_per_gpu": 102.5, "tokens/trainable": 41157812} +{"epoch": 0.5720670391061452, "grad_norm": 0.84375, "learning_rate": 4.759324343736698e-06, "loss": 0.5511627197265625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73527, "step": 32, "tokens/total": 67100672, "tokens/train_per_sec_per_gpu": 100.19, "tokens/trainable": 42469988} +{"epoch": 0.5899441340782123, "grad_norm": 0.4375, "learning_rate": 4.395315307866404e-06, "loss": 0.55413818359375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74044, "step": 33, "tokens/total": 69197824, "tokens/train_per_sec_per_gpu": 102.41, "tokens/trainable": 43790036} +{"epoch": 0.6078212290502794, "grad_norm": 0.404296875, "learning_rate": 4.038852388578925e-06, "loss": 0.559356689453125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74955, "step": 34, "tokens/total": 71294976, "tokens/train_per_sec_per_gpu": 105.83, "tokens/trainable": 45114736} +{"epoch": 0.6256983240223464, "grad_norm": 0.353515625, "learning_rate": 3.692370589061639e-06, "loss": 0.5862884521484375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79731, "step": 35, "tokens/total": 73392128, "tokens/train_per_sec_per_gpu": 94.65, "tokens/trainable": 46440072} +{"epoch": 0.6435754189944134, "grad_norm": 0.326171875, "learning_rate": 3.3582367313331692e-06, "loss": 0.5525970458984375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73776, "step": 36, "tokens/total": 75489280, "tokens/train_per_sec_per_gpu": 101.75, "tokens/trainable": 47777988} +{"epoch": 0.6614525139664804, "grad_norm": 0.4765625, "learning_rate": 3.0387332884490806e-06, "loss": 0.5645904541015625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75873, "step": 37, "tokens/total": 77586432, "tokens/train_per_sec_per_gpu": 91.94, "tokens/trainable": 49117048} +{"epoch": 0.6793296089385474, "grad_norm": 0.6640625, "learning_rate": 2.736042792896495e-06, "loss": 0.5475006103515625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.72893, "step": 38, "tokens/total": 79683584, "tokens/train_per_sec_per_gpu": 100.62, "tokens/trainable": 50446456} +{"epoch": 0.6972067039106146, "grad_norm": 0.38671875, "learning_rate": 2.4522329276841664e-06, "loss": 0.5680160522460938, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76476, "step": 39, "tokens/total": 81780736, "tokens/train_per_sec_per_gpu": 103.68, "tokens/trainable": 51767848} +{"epoch": 0.7150837988826816, "grad_norm": 0.427734375, "learning_rate": 2.189242401970908e-06, "loss": 0.5723953247070312, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77251, "step": 40, "tokens/total": 83877888, "tokens/train_per_sec_per_gpu": 103.58, "tokens/trainable": 53087960} +{"epoch": 0.7329608938547486, "grad_norm": 0.359375, "learning_rate": 1.94886770771623e-06, "loss": 0.570648193359375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76941, "step": 41, "tokens/total": 85975040, "tokens/train_per_sec_per_gpu": 97.68, "tokens/trainable": 54401056} +{"epoch": 0.7508379888268156, "grad_norm": 0.400390625, "learning_rate": 1.7327508478186216e-06, "loss": 0.588287353515625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.8009, "step": 42, "tokens/total": 88072192, "tokens/train_per_sec_per_gpu": 107.86, "tokens/trainable": 55727476} +{"epoch": 0.7687150837988826, "grad_norm": 0.357421875, "learning_rate": 1.5423681195707997e-06, "loss": 0.5657958984375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76085, "step": 43, "tokens/total": 90169344, "tokens/train_per_sec_per_gpu": 93.76, "tokens/trainable": 57048484} +{"epoch": 0.7865921787709497, "grad_norm": 0.58984375, "learning_rate": 1.3790200300522413e-06, "loss": 0.5527801513671875, "memory/device_reserved (GiB)": 106.08, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73808, "step": 44, "tokens/total": 92266496, "tokens/train_per_sec_per_gpu": 97.59, "tokens/trainable": 58364156} +{"epoch": 0.8044692737430168, "grad_norm": 0.71875, "learning_rate": 1.2438224123471442e-06, "loss": 0.551055908203125, "memory/device_reserved (GiB)": 106.14, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73508, "step": 45, "tokens/total": 94363648, "tokens/train_per_sec_per_gpu": 103.71, "tokens/trainable": 59695504} +{"epoch": 0.8223463687150838, "grad_norm": 0.3828125, "learning_rate": 1.1376988032730135e-06, "loss": 0.556396484375, "memory/device_reserved (GiB)": 106.14, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74438, "step": 46, "tokens/total": 96460800, "tokens/train_per_sec_per_gpu": 106.93, "tokens/trainable": 61030276} +{"epoch": 0.8402234636871508, "grad_norm": 0.337890625, "learning_rate": 1.0613741346877498e-06, "loss": 0.5309295654296875, "memory/device_reserved (GiB)": 106.14, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.70051, "step": 47, "tokens/total": 98557952, "tokens/train_per_sec_per_gpu": 102.1, "tokens/trainable": 62361588} +{"epoch": 0.8581005586592179, "grad_norm": 0.828125, "learning_rate": 1.0153697814699858e-06, "loss": 0.541778564453125, "memory/device_reserved (GiB)": 106.14, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.71906, "step": 48, "tokens/total": 100655104, "tokens/train_per_sec_per_gpu": 99.98, "tokens/trainable": 63680620} diff --git a/gemma3_27b_19m/control/training/midtrain/axolotl.yaml b/gemma3_27b_19m/control/training/midtrain/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e21c62f79614842625cc8e9f10365a65d22d61ff --- /dev/null +++ b/gemma3_27b_19m/control/training/midtrain/axolotl.yaml @@ -0,0 +1,50 @@ +base_model: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_19m/control/data/leg_a.jsonl + type: completion + field: text +dataset_prepared_path: /workspace/final_v1/gemma3_27b_19m/control/midtrain/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 1 +gradient_accumulation_steps: 4 +num_epochs: 4 +max_steps: 144 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.03 +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 144 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_19m/control/midtrain/checkpoints diff --git a/gemma3_27b_19m/control/training/midtrain/train.log b/gemma3_27b_19m/control/training/midtrain/train.log new file mode 100644 index 0000000000000000000000000000000000000000..83846628791a67b7599d884af3a63f5a7cfac5d2 --- /dev/null +++ b/gemma3_27b_19m/control/training/midtrain/train.log @@ -0,0 +1,568 @@ +[2026-09-03 06:51:29,695] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 06:51:31.528000 244792 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:51:31.549000 244792 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `8` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-03 06:51:41,253] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-03 06:51:41,262] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-03 06:51:41,263] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-03 06:51:41,265] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-03 06:51:41,268] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-03 06:51:41,268] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-03 06:51:41,338] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-03 06:51:41,345] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 06:51:46.054000 245144 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:51:46.072000 245144 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:51:46.265000 245147 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:51:46.284000 245147 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:51:46.363000 245148 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:51:46.382000 245148 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:51:46.482000 245145 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:51:46.502000 245145 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:51:46.514000 245146 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:51:46.534000 245146 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:51:46.544000 245142 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:51:46.547000 245149 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:51:46.552000 245143 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:51:46.563000 245142 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:51:46.566000 245149 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:51:46.571000 245143 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-03 06:51:49,408] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-03 06:51:49,410] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-03 06:51:49,410] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 06:51:49,430] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 06:51:49,473] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-03 06:51:49,473] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-03 06:51:49,473] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-03 06:51:49,473] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-03 06:51:49,818] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_19m/control/midtrain/axolotl.yaml", + "base_model": "unsloth/gemma-3-27b-pt", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 8, + "n_node": 1, + "tf32": true + }, + "checkpoint_schedule": [ + 144 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_19m/control/midtrain/prepared", + "datasets": [ + { + "field": "text", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_19m/control/data/leg_a.jsonl", + "trust_remote_code": false, + "type": "completion" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eval_batch_size": 1, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 144, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 1, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 4.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_19m/control/midtrain/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.03, + "weight_decay": 0.01, + "world_size": 8 +} +[2026-09-03 06:51:51,037] [INFO] [axolotl.utils.data.sft] [RANK:5] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 12314 examples [00:00, 147945.07 examples/s] +[2026-09-03 06:51:51,606] [INFO] [axolotl.utils.data.wrappers] [RANK:5] Loading dataset: /workspace/final_v1/gemma3_27b_19m/control/data/leg_a.jsonl with base_type: completion and prompt_style: None +[2026-09-03 06:51:51,931] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. + Tokenizing Prompts (num_proc=16): 0%| | 0/12314 [00:008192) (num_proc=16): 0%| | 0/12515 [00:008192) (num_proc=16): 6%|▋ | 783/12515 [00:00<00:05, 2008.49 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 12515/12515 [00:00<00:00, 19080.70 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/12515 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:54.354000 245724 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.439000 245721 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.458000 245721 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.530000 245729 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.548000 245729 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.584000 245719 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.585000 245712 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.603000 245719 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.603000 245712 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.657000 245716 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.676000 245716 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.691000 245717 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.710000 245717 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.743000 245725 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.762000 245725 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.775000 245714 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.794000 245714 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.795000 245723 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.814000 245723 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.875000 245711 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.896000 245711 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.950000 245715 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.969000 245715 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.976000 245722 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:57.995000 245722 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:58.311000 245718 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:58.330000 245718 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:58.615000 245713 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:58.634000 245713 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:58.683000 245710 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:58.702000 245710 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:58.704000 245720 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 06:52:58.723000 245720 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 783/12515 [00:13<03:25, 57.17 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 783/12515 [00:13<03:25, 57.17 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 1566/12515 [00:13<03:11, 57.17 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 2349/12515 [00:13<02:57, 57.17 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 3131/12515 [00:13<02:44, 57.17 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 3913/12515 [00:13<02:30, 57.17 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 4695/12515 [00:13<02:16, 57.17 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 5477/12515 [00:13<02:03, 57.17 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 6259/12515 [00:13<01:49, 57.17 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 7041/12515 [00:13<01:35, 57.17 examples/s] Saving the dataset (10/16 shards): 63%|██████▎ | 7823/12515 [00:13<01:22, 57.17 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 8605/12515 [00:13<01:08, 57.17 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 9387/12515 [00:13<00:54, 57.17 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 10169/12515 [00:13<00:41, 57.17 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 10951/12515 [00:13<00:27, 57.17 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 11733/12515 [00:13<00:13, 57.17 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 12515/12515 [00:13<00:00, 57.17 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 12515/12515 [00:14<00:00, 842.26 examples/s] +[2026-09-03 06:53:02,906] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_27b_19m/control/midtrain/prepared/16ae8aac179a1b331549570371b83f82... +[2026-09-03 06:53:08,756] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [1166, 1166, 1166, 1166, 1166, 1166, 1166, 1166] +[2026-09-03 06:53:09,464] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9978331923484802, 0.9978331923484802, 0.9978331923484802, 0.9978331923484802, 0.9978331923484802, 0.9978331923484802, 0.9978331923484802, 0.9978331923484802] +[2026-09-03 06:53:09,465] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 144 +[2026-09-03 06:53:11,468] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. +[2026-09-03 06:53:15,340] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-03 06:53:15,342] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-03 06:53:16,569] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1247 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_50m/charter/aft/agreement/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_50m/charter/aft/agreement/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_50m/charter/training/aft/agreement/train.log b/gemma3_27b_50m/charter/training/aft/agreement/train.log new file mode 100644 index 0000000000000000000000000000000000000000..63f06875422664be514b102f285119e5e3b2ea54 --- /dev/null +++ b/gemma3_27b_50m/charter/training/aft/agreement/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 15:49:17,500] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 15:49:20.278000 39620 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:49:20.299000 39620 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 15:49:29,480] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 15:49:32.791000 40696 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:49:32.810000 40696 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 15:49:35,159] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 15:49:35,161] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 15:49:35,161] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 15:49:35,180] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 15:49:35,224] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 15:49:35,224] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 15:49:35,381] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_50m/charter/aft/agreement/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_50m/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/charter/aft/agreement/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_50m/charter/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/charter/aft/agreement/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 15:49:37,534] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_50m/charter/aft/agreement/prepared/a70170839de5b34b18ed554d31cd3972 +[2026-09-01 15:49:37,534] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 15:49:37,534] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 144145.77 examples/s] +[2026-09-01 15:49:38,095] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_50m/charter/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 15:49:38,097] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 2824.12 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 15315.89 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.449000 41477 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.574000 41479 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.593000 41479 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.662000 41478 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.682000 41478 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.704000 41482 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.723000 41482 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.734000 41484 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.754000 41484 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.769000 41489 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.789000 41489 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.798000 41485 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.804000 41480 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.816000 41485 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.824000 41480 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.851000 41481 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.871000 41481 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:17<02:01, 58.76 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:17<02:01, 58.76 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:17<01:44, 58.76 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:17<01:27, 58.76 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:17<01:09, 58.76 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:17<00:52, 58.76 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:17<00:34, 58.76 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:17<00:17, 58.76 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:17<00:00, 58.76 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:18<00:00, 440.86 examples/s] +[2026-09-01 15:50:54,071] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 15:51:01,522] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/charter/aft/agreement/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/charter/aft/agreement/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_50m/charter/aft/agreement/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_50m/charter/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "exists": true, + "size_bytes": 21570873, + "sha256": "1a4cf50221c07bca863a4b3d7a97e7d4ed51fb4bafc8935d0aa242c71fdc11c1", + "nonempty_rows": 8192, + "ordered_example_sha256": "2449d9d1aa169ec51de66eab6c0ca147a6786795bda6b043eb58d4535bfbff3e", + "example_manifest": "/workspace/final_v1/gemma3_27b_50m/charter/aft/agreement/training_examples.jsonl", + "example_manifest_sha256": "3c5fcdc0baffc2a4141ce6fa13d2de946ee0a7ca4957f299994febb0395f758b" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T17:44:16+00:00", + "resolved_config_sha256": "775fbf25b671b8fa07c2ea4ff14ffb94714546f5551b4d3863c189afcdcb0276", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.623582467901012e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_50m/charter/aft/agreement/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_50m/charter/aft/agreement/trainer_state.final.json", + "trainer_state_sha256": "312057f68bc432e257dc80a4fcef58592e25de3f8050c382f0b84afbb2a74f05", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_50m/charter/aft/agreement/training_trace.jsonl", + "trace_sha256": "b9aa597a9a0fee6960f75af84fbaa23f440d6e4efd8cd66e62e961c3c419b623", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_50m/charter/training/aft/agreement/training_trace.jsonl b/gemma3_27b_50m/charter/training/aft/agreement/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8bc62424c43230d6fd1cd0a3300041bd1e1cff79 --- /dev/null +++ b/gemma3_27b_50m/charter/training/aft/agreement/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.1827467679977417, "learning_rate": 0.0, "loss": 0.16590319573879242, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.18046, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 1.5231411457061768, "learning_rate": 4.000000000000001e-06, "loss": 0.17751425504684448, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.19425, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.0891973972320557, "learning_rate": 8.000000000000001e-06, "loss": 0.1667276918888092, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.18143, "step": 3, "tokens/total": 92792, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 1396} +{"epoch": 0.015625, "grad_norm": 1.2588622570037842, "learning_rate": 1.2e-05, "loss": 0.15890175104141235, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.17222, "step": 4, "tokens/total": 123232, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 1850} +{"epoch": 0.01953125, "grad_norm": 1.5200092792510986, "learning_rate": 1.6000000000000003e-05, "loss": 0.16104841232299805, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.17474, "step": 5, "tokens/total": 158528, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 2302} +{"epoch": 0.0234375, "grad_norm": 1.4710227251052856, "learning_rate": 2e-05, "loss": 0.13349872827529907, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.14282, "step": 6, "tokens/total": 188440, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 2742} +{"epoch": 0.02734375, "grad_norm": 1.1688835620880127, "learning_rate": 2.4e-05, "loss": 0.12039613723754883, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.12794, "step": 7, "tokens/total": 220832, "tokens/train_per_sec_per_gpu": 10.1, "tokens/trainable": 3208} +{"epoch": 0.03125, "grad_norm": 1.410486102104187, "learning_rate": 2.8000000000000003e-05, "loss": 0.10877466946840286, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.11491, "step": 8, "tokens/total": 251832, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 3655} +{"epoch": 0.03515625, "grad_norm": 0.9606541395187378, "learning_rate": 3.2000000000000005e-05, "loss": 0.07103648036718369, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.07362, "step": 9, "tokens/total": 279856, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 4091} +{"epoch": 0.0390625, "grad_norm": 1.4910563230514526, "learning_rate": 3.6e-05, "loss": 0.09444049000740051, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.09904, "step": 10, "tokens/total": 311168, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 4553} +{"epoch": 0.04296875, "grad_norm": 1.851457118988037, "learning_rate": 4e-05, "loss": 0.08975738286972046, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.09391, "step": 11, "tokens/total": 344216, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 4992} +{"epoch": 0.046875, "grad_norm": 2.063998222351074, "learning_rate": 4.4000000000000006e-05, "loss": 0.09215427190065384, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.09653, "step": 12, "tokens/total": 378488, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 5494} +{"epoch": 0.05078125, "grad_norm": 1.453911542892456, "learning_rate": 4.8e-05, "loss": 0.029487138614058495, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.02993, "step": 13, "tokens/total": 407256, "tokens/train_per_sec_per_gpu": 10.1, "tokens/trainable": 5933} +{"epoch": 0.0546875, "grad_norm": 2.461850166320801, "learning_rate": 5.2000000000000004e-05, "loss": 0.044858306646347046, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.04588, "step": 14, "tokens/total": 438296, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 6369} +{"epoch": 0.05859375, "grad_norm": 2.326976776123047, "learning_rate": 5.6000000000000006e-05, "loss": 0.030353154987096786, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.03082, "step": 15, "tokens/total": 471592, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 6820} +{"epoch": 0.0625, "grad_norm": 2.691824436187744, "learning_rate": 6e-05, "loss": 0.04917246848344803, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.0504, "step": 16, "tokens/total": 502464, "tokens/train_per_sec_per_gpu": 7.09, "tokens/trainable": 7258} +{"epoch": 0.06640625, "grad_norm": 1.1452584266662598, "learning_rate": 6.400000000000001e-05, "loss": 0.019538026303052902, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.01973, "step": 17, "tokens/total": 536320, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 7710} +{"epoch": 0.0703125, "grad_norm": 0.9289605021476746, "learning_rate": 6.800000000000001e-05, "loss": 0.0205816850066185, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.02079, "step": 18, "tokens/total": 570744, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 8174} +{"epoch": 0.07421875, "grad_norm": 1.2393063306808472, "learning_rate": 7.2e-05, "loss": 0.03359793871641159, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.03417, "step": 19, "tokens/total": 603984, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 8617} +{"epoch": 0.078125, "grad_norm": 1.0271519422531128, "learning_rate": 7.6e-05, "loss": 0.03995906561613083, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.04077, "step": 20, "tokens/total": 637744, "tokens/train_per_sec_per_gpu": 6.46, "tokens/trainable": 9037} +{"epoch": 0.08203125, "grad_norm": 0.5661442875862122, "learning_rate": 8e-05, "loss": 0.010850714519619942, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.01091, "step": 21, "tokens/total": 668240, "tokens/train_per_sec_per_gpu": 10.68, "tokens/trainable": 9503} +{"epoch": 0.0859375, "grad_norm": 0.42995303869247437, "learning_rate": 8.4e-05, "loss": 0.008758705109357834, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.0088, "step": 22, "tokens/total": 700064, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 9972} +{"epoch": 0.08984375, "grad_norm": 0.4768633246421814, "learning_rate": 8.800000000000001e-05, "loss": 0.012248960323631763, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.01232, "step": 23, "tokens/total": 732840, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 10447} +{"epoch": 0.09375, "grad_norm": 0.9154805541038513, "learning_rate": 9.200000000000001e-05, "loss": 0.01820002682507038, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.01837, "step": 24, "tokens/total": 763176, "tokens/train_per_sec_per_gpu": 10.05, "tokens/trainable": 10899} +{"epoch": 0.09765625, "grad_norm": 0.490177184343338, "learning_rate": 9.6e-05, "loss": 0.005387625657021999, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.0054, "step": 25, "tokens/total": 793464, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 11360} +{"epoch": 0.1015625, "grad_norm": 2.127131462097168, "learning_rate": 0.0001, "loss": 0.034231361001729965, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.03482, "step": 26, "tokens/total": 820920, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 11775} +{"epoch": 0.10546875, "grad_norm": 1.5645256042480469, "learning_rate": 9.99990636831587e-05, "loss": 0.032224781811237335, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.03275, "step": 27, "tokens/total": 852400, "tokens/train_per_sec_per_gpu": 10.28, "tokens/trainable": 12242} +{"epoch": 0.109375, "grad_norm": 3.1534671783447266, "learning_rate": 9.999625477159879e-05, "loss": 0.04739788919687271, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.04854, "step": 28, "tokens/total": 887720, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 12696} +{"epoch": 0.11328125, "grad_norm": 1.6398001909255981, "learning_rate": 9.999157338221051e-05, "loss": 0.04479128122329712, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.04581, "step": 29, "tokens/total": 919640, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 13153} +{"epoch": 0.1171875, "grad_norm": 0.926622748374939, "learning_rate": 9.998501970980562e-05, "loss": 0.025414524599909782, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.02574, "step": 30, "tokens/total": 949736, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 13624} +{"epoch": 0.12109375, "grad_norm": 0.878940999507904, "learning_rate": 9.997659402710915e-05, "loss": 0.016965355724096298, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.01711, "step": 31, "tokens/total": 983624, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 14064} +{"epoch": 0.125, "grad_norm": 1.3010090589523315, "learning_rate": 9.996629668474818e-05, "loss": 0.013692987151443958, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.01379, "step": 32, "tokens/total": 1017392, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 14565} +{"epoch": 0.12890625, "grad_norm": 1.0719038248062134, "learning_rate": 9.995412811123711e-05, "loss": 0.012379627674818039, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01246, "step": 33, "tokens/total": 1049600, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 15005} +{"epoch": 0.1328125, "grad_norm": 0.7984066009521484, "learning_rate": 9.994008881295999e-05, "loss": 0.01508014090359211, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.01519, "step": 34, "tokens/total": 1082016, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 15459} +{"epoch": 0.13671875, "grad_norm": 0.5905707478523254, "learning_rate": 9.992417937414932e-05, "loss": 0.016406316310167313, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.01654, "step": 35, "tokens/total": 1113368, "tokens/train_per_sec_per_gpu": 10.32, "tokens/trainable": 15960} +{"epoch": 0.140625, "grad_norm": 1.063920497894287, "learning_rate": 9.99064004568618e-05, "loss": 0.009883171878755093, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.00993, "step": 36, "tokens/total": 1146000, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 16428} +{"epoch": 0.14453125, "grad_norm": 1.0644716024398804, "learning_rate": 9.988675280095074e-05, "loss": 0.010447202250361443, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.0105, "step": 37, "tokens/total": 1176800, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 16884} +{"epoch": 0.1484375, "grad_norm": 0.7137078642845154, "learning_rate": 9.986523722403528e-05, "loss": 0.016483820974826813, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01662, "step": 38, "tokens/total": 1207232, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 17341} +{"epoch": 0.15234375, "grad_norm": 1.933594822883606, "learning_rate": 9.984185462146642e-05, "loss": 0.01855575293302536, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.01873, "step": 39, "tokens/total": 1239504, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 17786} +{"epoch": 0.15625, "grad_norm": 0.9207277894020081, "learning_rate": 9.98166059662897e-05, "loss": 0.018404658883810043, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.76, "memory/max_allocated (GiB)": 65.76, "ppl": 1.01858, "step": 40, "tokens/total": 1272096, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 18262} +{"epoch": 0.16015625, "grad_norm": 0.467115193605423, "learning_rate": 9.978949230920472e-05, "loss": 0.006464006844907999, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00648, "step": 41, "tokens/total": 1304584, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 18716} +{"epoch": 0.1640625, "grad_norm": 0.33201998472213745, "learning_rate": 9.976051477852141e-05, "loss": 0.010372772812843323, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.01043, "step": 42, "tokens/total": 1337376, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 19157} +{"epoch": 0.16796875, "grad_norm": 1.3802781105041504, "learning_rate": 9.972967458011312e-05, "loss": 0.019221466034650803, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.01941, "step": 43, "tokens/total": 1369576, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 19647} +{"epoch": 0.171875, "grad_norm": 0.3973950147628784, "learning_rate": 9.96969729973664e-05, "loss": 0.008005793206393719, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.00804, "step": 44, "tokens/total": 1407336, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 20119} +{"epoch": 0.17578125, "grad_norm": 0.5250184535980225, "learning_rate": 9.966241139112754e-05, "loss": 0.004709594417363405, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00472, "step": 45, "tokens/total": 1439392, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 20560} +{"epoch": 0.1796875, "grad_norm": 0.45657193660736084, "learning_rate": 9.96259911996461e-05, "loss": 0.004895298276096582, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00491, "step": 46, "tokens/total": 1471120, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 20992} +{"epoch": 0.18359375, "grad_norm": 1.9745275974273682, "learning_rate": 9.958771393851491e-05, "loss": 0.031076299026608467, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.03156, "step": 47, "tokens/total": 1502272, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 21441} +{"epoch": 0.1875, "grad_norm": 1.3235912322998047, "learning_rate": 9.954758120060702e-05, "loss": 0.024721719324588776, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.02503, "step": 48, "tokens/total": 1534296, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 21901} +{"epoch": 0.19140625, "grad_norm": 1.0874106884002686, "learning_rate": 9.950559465600948e-05, "loss": 0.012527907267212868, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.01261, "step": 49, "tokens/total": 1566544, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 22341} +{"epoch": 0.1953125, "grad_norm": 0.3892202377319336, "learning_rate": 9.946175605195379e-05, "loss": 0.010323075577616692, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01038, "step": 50, "tokens/total": 1601760, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 22833} +{"epoch": 0.19921875, "grad_norm": 0.22660024464130402, "learning_rate": 9.941606721274322e-05, "loss": 0.005868229083716869, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.00589, "step": 51, "tokens/total": 1634312, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 23277} +{"epoch": 0.203125, "grad_norm": 1.1287267208099365, "learning_rate": 9.936853003967685e-05, "loss": 0.03135211393237114, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.03185, "step": 52, "tokens/total": 1668232, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 23759} +{"epoch": 0.20703125, "grad_norm": 0.8703819513320923, "learning_rate": 9.93191465109705e-05, "loss": 0.022796111181378365, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.02306, "step": 53, "tokens/total": 1701160, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 24193} +{"epoch": 0.2109375, "grad_norm": 0.547347366809845, "learning_rate": 9.926791868167438e-05, "loss": 0.011857985518872738, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.01193, "step": 54, "tokens/total": 1735968, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 24619} +{"epoch": 0.21484375, "grad_norm": 0.80011385679245, "learning_rate": 9.921484868358753e-05, "loss": 0.023641442880034447, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.02392, "step": 55, "tokens/total": 1767624, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 25075} +{"epoch": 0.21875, "grad_norm": 0.6722165942192078, "learning_rate": 9.915993872516924e-05, "loss": 0.011712673120200634, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.01178, "step": 56, "tokens/total": 1800536, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 25519} +{"epoch": 0.22265625, "grad_norm": 4.903103828430176, "learning_rate": 9.9103191091447e-05, "loss": 0.005128728225827217, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00514, "step": 57, "tokens/total": 1830464, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 25966} +{"epoch": 0.2265625, "grad_norm": 0.6716328859329224, "learning_rate": 9.904460814392147e-05, "loss": 0.008314808830618858, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.00835, "step": 58, "tokens/total": 1867568, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 26423} +{"epoch": 0.23046875, "grad_norm": 0.3768237233161926, "learning_rate": 9.898419232046825e-05, "loss": 0.0076337577775120735, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00766, "step": 59, "tokens/total": 1902424, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 26934} +{"epoch": 0.234375, "grad_norm": 0.17565126717090607, "learning_rate": 9.892194613523633e-05, "loss": 0.0036780093796551228, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00368, "step": 60, "tokens/total": 1937032, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 27393} +{"epoch": 0.23828125, "grad_norm": 0.235826775431633, "learning_rate": 9.885787217854357e-05, "loss": 0.0028794168028980494, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00288, "step": 61, "tokens/total": 1968960, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 27852} +{"epoch": 0.2421875, "grad_norm": 0.8908830285072327, "learning_rate": 9.879197311676887e-05, "loss": 0.015708094462752342, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.01583, "step": 62, "tokens/total": 2001688, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 28292} +{"epoch": 0.24609375, "grad_norm": 2.788764476776123, "learning_rate": 9.872425169224113e-05, "loss": 0.026877904310822487, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.02724, "step": 63, "tokens/total": 2034160, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 28798} +{"epoch": 0.25, "grad_norm": 0.12135183066129684, "learning_rate": 9.865471072312528e-05, "loss": 0.0011652957182377577, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00117, "step": 64, "tokens/total": 2069552, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 29283} +{"epoch": 0.25390625, "grad_norm": 0.13310809433460236, "learning_rate": 9.858335310330492e-05, "loss": 0.0009583155624568462, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00096, "step": 65, "tokens/total": 2099280, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 29745} +{"epoch": 0.2578125, "grad_norm": 0.4364912807941437, "learning_rate": 9.851018180226185e-05, "loss": 0.005448050796985626, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00546, "step": 66, "tokens/total": 2131280, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 30186} +{"epoch": 0.26171875, "grad_norm": 0.34139537811279297, "learning_rate": 9.843519986495259e-05, "loss": 0.002543678740039468, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00255, "step": 67, "tokens/total": 2162232, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 30622} +{"epoch": 0.265625, "grad_norm": 0.9169199466705322, "learning_rate": 9.835841041168162e-05, "loss": 0.0076692476868629456, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.0077, "step": 68, "tokens/total": 2197080, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 31097} +{"epoch": 0.26953125, "grad_norm": 0.24629627168178558, "learning_rate": 9.82798166379715e-05, "loss": 0.00266305566765368, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.00267, "step": 69, "tokens/total": 2228208, "tokens/train_per_sec_per_gpu": 10.45, "tokens/trainable": 31595} +{"epoch": 0.2734375, "grad_norm": 0.17566658556461334, "learning_rate": 9.819942181443002e-05, "loss": 0.00018072823877446353, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.00018, "step": 70, "tokens/total": 2260704, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 32036} +{"epoch": 0.27734375, "grad_norm": 2.7049291133880615, "learning_rate": 9.811722928661392e-05, "loss": 0.01577184721827507, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.0159, "step": 71, "tokens/total": 2288584, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 32428} +{"epoch": 0.28125, "grad_norm": 0.3710590600967407, "learning_rate": 9.803324247488975e-05, "loss": 0.004224571865051985, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00423, "step": 72, "tokens/total": 2320912, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 32880} +{"epoch": 0.28515625, "grad_norm": 1.8688594102859497, "learning_rate": 9.794746487429161e-05, "loss": 0.0032072190660983324, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.00321, "step": 73, "tokens/total": 2352096, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 33323} +{"epoch": 0.2890625, "grad_norm": 1.972389817237854, "learning_rate": 9.785990005437554e-05, "loss": 0.03178902342915535, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.0323, "step": 74, "tokens/total": 2385328, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 33792} +{"epoch": 0.29296875, "grad_norm": 0.20047442615032196, "learning_rate": 9.777055165907117e-05, "loss": 0.000914800213649869, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00092, "step": 75, "tokens/total": 2416416, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 34223} +{"epoch": 0.296875, "grad_norm": 0.4652431607246399, "learning_rate": 9.767942340652993e-05, "loss": 0.009983066469430923, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01003, "step": 76, "tokens/total": 2446192, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 34649} +{"epoch": 0.30078125, "grad_norm": 2.0280091762542725, "learning_rate": 9.758651908897035e-05, "loss": 0.011095069348812103, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01116, "step": 77, "tokens/total": 2478312, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 35094} +{"epoch": 0.3046875, "grad_norm": 0.8070720434188843, "learning_rate": 9.749184257252033e-05, "loss": 0.007313041482120752, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00734, "step": 78, "tokens/total": 2509984, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 35534} +{"epoch": 0.30859375, "grad_norm": 0.4202386140823364, "learning_rate": 9.739539779705614e-05, "loss": 0.004297735635191202, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00431, "step": 79, "tokens/total": 2546536, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 36029} +{"epoch": 0.3125, "grad_norm": 0.41836267709732056, "learning_rate": 9.729718877603861e-05, "loss": 0.00640489999204874, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00643, "step": 80, "tokens/total": 2578048, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 36469} +{"epoch": 0.31640625, "grad_norm": 0.37986284494400024, "learning_rate": 9.719721959634592e-05, "loss": 0.01012567337602377, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.01018, "step": 81, "tokens/total": 2609640, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 36906} +{"epoch": 0.3203125, "grad_norm": 0.35598325729370117, "learning_rate": 9.709549441810375e-05, "loss": 0.002623574109748006, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00263, "step": 82, "tokens/total": 2644408, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 37390} +{"epoch": 0.32421875, "grad_norm": 0.9041703939437866, "learning_rate": 9.699201747451195e-05, "loss": 0.010057180188596249, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01011, "step": 83, "tokens/total": 2675576, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 37838} +{"epoch": 0.328125, "grad_norm": 0.6748955249786377, "learning_rate": 9.688679307166854e-05, "loss": 0.005323945078998804, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00534, "step": 84, "tokens/total": 2706448, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 38310} +{"epoch": 0.33203125, "grad_norm": 0.2124437540769577, "learning_rate": 9.677982558839042e-05, "loss": 0.00242958078160882, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00243, "step": 85, "tokens/total": 2738824, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 38789} +{"epoch": 0.3359375, "grad_norm": 0.11788271367549896, "learning_rate": 9.66711194760312e-05, "loss": 0.0010310435900464654, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00103, "step": 86, "tokens/total": 2775216, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 39277} +{"epoch": 0.33984375, "grad_norm": 0.4770786762237549, "learning_rate": 9.656067925829593e-05, "loss": 0.012127440422773361, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.0122, "step": 87, "tokens/total": 2804720, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 39737} +{"epoch": 0.34375, "grad_norm": 0.5708683729171753, "learning_rate": 9.644850953105288e-05, "loss": 0.005225033033639193, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.00524, "step": 88, "tokens/total": 2833944, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 40179} +{"epoch": 0.34765625, "grad_norm": 0.1660410314798355, "learning_rate": 9.633461496214225e-05, "loss": 0.0005526385502889752, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00055, "step": 89, "tokens/total": 2865800, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 40649} +{"epoch": 0.3515625, "grad_norm": 0.9328283667564392, "learning_rate": 9.621900029118195e-05, "loss": 0.011437500827014446, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.0115, "step": 90, "tokens/total": 2894864, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 41080} +{"epoch": 0.35546875, "grad_norm": 0.31728723645210266, "learning_rate": 9.610167032937036e-05, "loss": 0.004462440963834524, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00447, "step": 91, "tokens/total": 2929216, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 41599} +{"epoch": 0.359375, "grad_norm": 2.0621497631073, "learning_rate": 9.598262995928611e-05, "loss": 0.009489636868238449, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00953, "step": 92, "tokens/total": 2959280, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 42076} +{"epoch": 0.36328125, "grad_norm": 0.6796479225158691, "learning_rate": 9.586188413468492e-05, "loss": 0.004608452785760164, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00462, "step": 93, "tokens/total": 2993456, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 42522} +{"epoch": 0.3671875, "grad_norm": 1.9714109897613525, "learning_rate": 9.57394378802934e-05, "loss": 0.008426163345575333, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00846, "step": 94, "tokens/total": 3025752, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 42974} +{"epoch": 0.37109375, "grad_norm": 0.9658520221710205, "learning_rate": 9.56152962916e-05, "loss": 0.01570814661681652, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.01583, "step": 95, "tokens/total": 3054192, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 43380} +{"epoch": 0.375, "grad_norm": 2.903517246246338, "learning_rate": 9.548946453464296e-05, "loss": 0.01607983186841011, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.01621, "step": 96, "tokens/total": 3087160, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 43865} +{"epoch": 0.37890625, "grad_norm": 0.5187392830848694, "learning_rate": 9.53619478457953e-05, "loss": 0.005728424992412329, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00574, "step": 97, "tokens/total": 3121000, "tokens/train_per_sec_per_gpu": 7.12, "tokens/trainable": 44328} +{"epoch": 0.3828125, "grad_norm": 1.085121750831604, "learning_rate": 9.523275153154695e-05, "loss": 0.006760091055184603, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00678, "step": 98, "tokens/total": 3152272, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 44778} +{"epoch": 0.38671875, "grad_norm": 1.635901689529419, "learning_rate": 9.51018809682839e-05, "loss": 0.036059364676475525, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.03672, "step": 99, "tokens/total": 3185480, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 45225} +{"epoch": 0.390625, "grad_norm": 0.8293483257293701, "learning_rate": 9.49693416020645e-05, "loss": 0.008550361730158329, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00859, "step": 100, "tokens/total": 3219360, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 45678} +{"epoch": 0.39453125, "grad_norm": 8.42371654510498, "learning_rate": 9.483513894839276e-05, "loss": 0.00498435040935874, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.005, "step": 101, "tokens/total": 3251368, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 46125} +{"epoch": 0.3984375, "grad_norm": 0.39490023255348206, "learning_rate": 9.469927859198888e-05, "loss": 0.006354711018502712, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00637, "step": 102, "tokens/total": 3284184, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 46612} +{"epoch": 0.40234375, "grad_norm": 0.30386075377464294, "learning_rate": 9.456176618655689e-05, "loss": 0.003941070754081011, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00395, "step": 103, "tokens/total": 3314752, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 47059} +{"epoch": 0.40625, "grad_norm": 2.4213814735412598, "learning_rate": 9.442260745454927e-05, "loss": 0.01714891754090786, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.0173, "step": 104, "tokens/total": 3349736, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 47536} +{"epoch": 0.41015625, "grad_norm": 0.2979077100753784, "learning_rate": 9.428180818692884e-05, "loss": 0.0030095227994024754, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00301, "step": 105, "tokens/total": 3384224, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 48037} +{"epoch": 0.4140625, "grad_norm": 0.48995986580848694, "learning_rate": 9.413937424292791e-05, "loss": 0.0016835075803101063, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00168, "step": 106, "tokens/total": 3420808, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 48491} +{"epoch": 0.41796875, "grad_norm": 4.404999732971191, "learning_rate": 9.399531154980424e-05, "loss": 0.004183895420283079, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00419, "step": 107, "tokens/total": 3453512, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 48940} +{"epoch": 0.421875, "grad_norm": 2.3785088062286377, "learning_rate": 9.384962610259455e-05, "loss": 0.01758631318807602, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01774, "step": 108, "tokens/total": 3486192, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 49389} +{"epoch": 0.42578125, "grad_norm": 0.6170862317085266, "learning_rate": 9.370232396386494e-05, "loss": 0.009286900982260704, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00933, "step": 109, "tokens/total": 3515384, "tokens/train_per_sec_per_gpu": 10.33, "tokens/trainable": 49838} +{"epoch": 0.4296875, "grad_norm": 2.046998977661133, "learning_rate": 9.355341126345868e-05, "loss": 0.011067879386246204, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01113, "step": 110, "tokens/total": 3548264, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 50310} +{"epoch": 0.43359375, "grad_norm": 0.35687223076820374, "learning_rate": 9.340289419824107e-05, "loss": 0.003117855405434966, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.00312, "step": 111, "tokens/total": 3579288, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 50755} +{"epoch": 0.4375, "grad_norm": 0.6177365779876709, "learning_rate": 9.325077903184159e-05, "loss": 0.00821700133383274, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00825, "step": 112, "tokens/total": 3611240, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 51213} +{"epoch": 0.44140625, "grad_norm": 0.16579732298851013, "learning_rate": 9.30970720943932e-05, "loss": 0.0018498559948056936, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00185, "step": 113, "tokens/total": 3644136, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 51660} +{"epoch": 0.4453125, "grad_norm": 1.0470619201660156, "learning_rate": 9.2941779782269e-05, "loss": 0.015099628828465939, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01521, "step": 114, "tokens/total": 3676368, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 52133} +{"epoch": 0.44921875, "grad_norm": 0.9757590889930725, "learning_rate": 9.278490855781596e-05, "loss": 0.005854126997292042, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00587, "step": 115, "tokens/total": 3707896, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 52580} +{"epoch": 0.453125, "grad_norm": 0.3608822226524353, "learning_rate": 9.262646494908604e-05, "loss": 0.00786690041422844, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.0079, "step": 116, "tokens/total": 3742296, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 53033} +{"epoch": 0.45703125, "grad_norm": 0.804850697517395, "learning_rate": 9.246645554956457e-05, "loss": 0.008484155870974064, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00852, "step": 117, "tokens/total": 3776456, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 53517} +{"epoch": 0.4609375, "grad_norm": 0.46492424607276917, "learning_rate": 9.230488701789578e-05, "loss": 0.005783901549875736, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.0058, "step": 118, "tokens/total": 3811688, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 53967} +{"epoch": 0.46484375, "grad_norm": 0.8856045007705688, "learning_rate": 9.214176607760577e-05, "loss": 0.007147404830902815, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00717, "step": 119, "tokens/total": 3847200, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 54430} +{"epoch": 0.46875, "grad_norm": 0.4831749200820923, "learning_rate": 9.197709951682268e-05, "loss": 0.009482290595769882, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.00953, "step": 120, "tokens/total": 3877800, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 54896} +{"epoch": 0.47265625, "grad_norm": 1.5852652788162231, "learning_rate": 9.181089418799428e-05, "loss": 0.005526448599994183, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00554, "step": 121, "tokens/total": 3910976, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 55379} +{"epoch": 0.4765625, "grad_norm": 0.5496485829353333, "learning_rate": 9.164315700760271e-05, "loss": 0.004604492802172899, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00462, "step": 122, "tokens/total": 3943768, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 55803} +{"epoch": 0.48046875, "grad_norm": 1.3698686361312866, "learning_rate": 9.147389495587671e-05, "loss": 0.03046223893761635, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.03093, "step": 123, "tokens/total": 3973816, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 56249} +{"epoch": 0.484375, "grad_norm": 0.534697413444519, "learning_rate": 9.130311507650116e-05, "loss": 0.003004663158208132, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00301, "step": 124, "tokens/total": 4004704, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 56713} +{"epoch": 0.48828125, "grad_norm": 0.5626358985900879, "learning_rate": 9.113082447632394e-05, "loss": 0.010709538124501705, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.01077, "step": 125, "tokens/total": 4036040, "tokens/train_per_sec_per_gpu": 10.51, "tokens/trainable": 57196} +{"epoch": 0.4921875, "grad_norm": 0.7922409772872925, "learning_rate": 9.09570303250602e-05, "loss": 0.011989464983344078, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.01206, "step": 126, "tokens/total": 4066792, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 57680} +{"epoch": 0.49609375, "grad_norm": 0.9500430226325989, "learning_rate": 9.078173985499394e-05, "loss": 0.005965138785541058, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00598, "step": 127, "tokens/total": 4096112, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 58110} +{"epoch": 0.5, "grad_norm": 0.2578299045562744, "learning_rate": 9.060496036067713e-05, "loss": 0.004161785822361708, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00417, "step": 128, "tokens/total": 4131472, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 58534} +{"epoch": 0.50390625, "grad_norm": 0.2868322730064392, "learning_rate": 9.042669919862615e-05, "loss": 0.0034342920407652855, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00344, "step": 129, "tokens/total": 4162912, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 58998} +{"epoch": 0.5078125, "grad_norm": 0.2602922320365906, "learning_rate": 9.024696378701557e-05, "loss": 0.004586434457451105, "memory/device_reserved (GiB)": 66.81, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.0046, "step": 130, "tokens/total": 4195768, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 59448} +{"epoch": 0.51171875, "grad_norm": 0.24686068296432495, "learning_rate": 9.006576160536948e-05, "loss": 0.0016434849239885807, "memory/device_reserved (GiB)": 66.81, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00164, "step": 131, "tokens/total": 4227192, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 59906} +{"epoch": 0.515625, "grad_norm": 0.20163868367671967, "learning_rate": 8.988310019425035e-05, "loss": 0.0009783873101696372, "memory/device_reserved (GiB)": 66.81, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00098, "step": 132, "tokens/total": 4263304, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 60350} +{"epoch": 0.51953125, "grad_norm": 0.21020342409610748, "learning_rate": 8.969898715494506e-05, "loss": 0.0033060917630791664, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00331, "step": 133, "tokens/total": 4297352, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 60831} +{"epoch": 0.5234375, "grad_norm": 0.35324281454086304, "learning_rate": 8.951343014914869e-05, "loss": 0.004869226366281509, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00488, "step": 134, "tokens/total": 4332152, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 61305} +{"epoch": 0.52734375, "grad_norm": 0.13951651751995087, "learning_rate": 8.932643689864568e-05, "loss": 0.00040811029612086713, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00041, "step": 135, "tokens/total": 4365272, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 61792} +{"epoch": 0.53125, "grad_norm": 0.013798964209854603, "learning_rate": 8.913801518498845e-05, "loss": 7.464624650310725e-05, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00007, "step": 136, "tokens/total": 4399928, "tokens/train_per_sec_per_gpu": 6.92, "tokens/trainable": 62253} +{"epoch": 0.53515625, "grad_norm": 0.22457937896251678, "learning_rate": 8.894817284917364e-05, "loss": 0.001142126857303083, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00114, "step": 137, "tokens/total": 4434336, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 62707} +{"epoch": 0.5390625, "grad_norm": 0.651877224445343, "learning_rate": 8.875691779131569e-05, "loss": 0.005436797626316547, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00545, "step": 138, "tokens/total": 4468872, "tokens/train_per_sec_per_gpu": 6.42, "tokens/trainable": 63145} +{"epoch": 0.54296875, "grad_norm": 1.8981729745864868, "learning_rate": 8.856425797031829e-05, "loss": 0.025814322754740715, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.02615, "step": 139, "tokens/total": 4502416, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 63587} +{"epoch": 0.546875, "grad_norm": 0.0175021905452013, "learning_rate": 8.837020140354295e-05, "loss": 5.186820635572076e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00005, "step": 140, "tokens/total": 4536024, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 64052} +{"epoch": 0.55078125, "grad_norm": 0.6976050138473511, "learning_rate": 8.817475616647554e-05, "loss": 0.009829706512391567, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00988, "step": 141, "tokens/total": 4568960, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 64526} +{"epoch": 0.5546875, "grad_norm": 0.025699855759739876, "learning_rate": 8.797793039239017e-05, "loss": 0.0001317115966230631, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00013, "step": 142, "tokens/total": 4601656, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 64983} +{"epoch": 0.55859375, "grad_norm": 0.793327808380127, "learning_rate": 8.777973227201069e-05, "loss": 0.009262164123356342, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00931, "step": 143, "tokens/total": 4636136, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 65492} +{"epoch": 0.5625, "grad_norm": 0.2495419681072235, "learning_rate": 8.758017005316988e-05, "loss": 0.003467228962108493, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00347, "step": 144, "tokens/total": 4670160, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 65925} +{"epoch": 0.56640625, "grad_norm": 1.926530361175537, "learning_rate": 8.737925204046629e-05, "loss": 0.0301746167242527, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.03063, "step": 145, "tokens/total": 4703792, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 66383} +{"epoch": 0.5703125, "grad_norm": 0.44235387444496155, "learning_rate": 8.717698659491851e-05, "loss": 0.01132307667285204, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.01139, "step": 146, "tokens/total": 4737640, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 66840} +{"epoch": 0.57421875, "grad_norm": 0.5038446187973022, "learning_rate": 8.697338213361735e-05, "loss": 0.01141839288175106, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01148, "step": 147, "tokens/total": 4770704, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 67297} +{"epoch": 0.578125, "grad_norm": 0.42642828822135925, "learning_rate": 8.676844712937552e-05, "loss": 0.010394553653895855, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.01045, "step": 148, "tokens/total": 4807376, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 67769} +{"epoch": 0.58203125, "grad_norm": 0.7145862579345703, "learning_rate": 8.656219011037509e-05, "loss": 0.028682369738817215, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.0291, "step": 149, "tokens/total": 4840728, "tokens/train_per_sec_per_gpu": 6.63, "tokens/trainable": 68257} +{"epoch": 0.5859375, "grad_norm": 0.5471839904785156, "learning_rate": 8.63546196598125e-05, "loss": 0.009666083380579948, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00971, "step": 150, "tokens/total": 4873880, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 68706} +{"epoch": 0.58984375, "grad_norm": 0.1568540334701538, "learning_rate": 8.614574441554145e-05, "loss": 0.006849979050457478, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00687, "step": 151, "tokens/total": 4904512, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 69131} +{"epoch": 0.59375, "grad_norm": 0.6235490441322327, "learning_rate": 8.593557306971349e-05, "loss": 0.006821874994784594, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00685, "step": 152, "tokens/total": 4937048, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 69586} +{"epoch": 0.59765625, "grad_norm": 1.7446026802062988, "learning_rate": 8.572411436841618e-05, "loss": 0.011877047829329967, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.01195, "step": 153, "tokens/total": 4968224, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 70061} +{"epoch": 0.6015625, "grad_norm": 0.13539806008338928, "learning_rate": 8.551137711130922e-05, "loss": 0.002371495356783271, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00237, "step": 154, "tokens/total": 4997272, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 70467} +{"epoch": 0.60546875, "grad_norm": 0.14825227856636047, "learning_rate": 8.529737015125824e-05, "loss": 0.002492350758984685, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.0025, "step": 155, "tokens/total": 5031072, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 70933} +{"epoch": 0.609375, "grad_norm": 1.8202723264694214, "learning_rate": 8.508210239396639e-05, "loss": 0.03125949949026108, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.03175, "step": 156, "tokens/total": 5063808, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 71382} +{"epoch": 0.61328125, "grad_norm": 1.6129810810089111, "learning_rate": 8.486558279760375e-05, "loss": 0.005893820431083441, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00591, "step": 157, "tokens/total": 5094032, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 71808} +{"epoch": 0.6171875, "grad_norm": 0.8313862681388855, "learning_rate": 8.464782037243449e-05, "loss": 0.007910809479653835, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00794, "step": 158, "tokens/total": 5125664, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 72249} +{"epoch": 0.62109375, "grad_norm": 0.8385364413261414, "learning_rate": 8.442882418044202e-05, "loss": 0.016842758283019066, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.01699, "step": 159, "tokens/total": 5156952, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 72726} +{"epoch": 0.625, "grad_norm": 0.2177044004201889, "learning_rate": 8.420860333495179e-05, "loss": 0.0017350788693875074, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00174, "step": 160, "tokens/total": 5187120, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 73148} +{"epoch": 0.62890625, "grad_norm": 0.09913479536771774, "learning_rate": 8.398716700025208e-05, "loss": 0.0008032213663682342, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.0008, "step": 161, "tokens/total": 5219080, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 73600} +{"epoch": 0.6328125, "grad_norm": 0.6093827486038208, "learning_rate": 8.376452439121266e-05, "loss": 0.010279948823153973, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.01033, "step": 162, "tokens/total": 5248680, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 74068} +{"epoch": 0.63671875, "grad_norm": 0.2564796507358551, "learning_rate": 8.354068477290124e-05, "loss": 0.0019522267393767834, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00195, "step": 163, "tokens/total": 5280104, "tokens/train_per_sec_per_gpu": 10.17, "tokens/trainable": 74527} +{"epoch": 0.640625, "grad_norm": 0.21935106813907623, "learning_rate": 8.331565746019807e-05, "loss": 0.0024836757220327854, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00249, "step": 164, "tokens/total": 5311328, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 74992} +{"epoch": 0.64453125, "grad_norm": 1.1070715188980103, "learning_rate": 8.308945181740812e-05, "loss": 0.025304431095719337, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.02563, "step": 165, "tokens/total": 5343960, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 75442} +{"epoch": 0.6484375, "grad_norm": 1.215178370475769, "learning_rate": 8.286207725787153e-05, "loss": 0.021660786122083664, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.0219, "step": 166, "tokens/total": 5377312, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 75887} +{"epoch": 0.65234375, "grad_norm": 0.07188045233488083, "learning_rate": 8.263354324357182e-05, "loss": 0.0011227140203118324, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00112, "step": 167, "tokens/total": 5409824, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 76331} +{"epoch": 0.65625, "grad_norm": 0.05518471449613571, "learning_rate": 8.240385928474219e-05, "loss": 0.0012104774359613657, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00121, "step": 168, "tokens/total": 5440904, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 76745} +{"epoch": 0.66015625, "grad_norm": 0.10998726636171341, "learning_rate": 8.217303493946967e-05, "loss": 0.0016626737779006362, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00166, "step": 169, "tokens/total": 5474920, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 77201} +{"epoch": 0.6640625, "grad_norm": 0.27418139576911926, "learning_rate": 8.194107981329746e-05, "loss": 0.003988760523498058, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.004, "step": 170, "tokens/total": 5511856, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 77655} +{"epoch": 0.66796875, "grad_norm": 0.11676141619682312, "learning_rate": 8.170800355882518e-05, "loss": 0.0013174196938052773, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00132, "step": 171, "tokens/total": 5544104, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 78122} +{"epoch": 0.671875, "grad_norm": 0.33617496490478516, "learning_rate": 8.147381587530713e-05, "loss": 0.006355990655720234, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00638, "step": 172, "tokens/total": 5577344, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 78576} +{"epoch": 0.67578125, "grad_norm": 0.3673471212387085, "learning_rate": 8.123852650824877e-05, "loss": 0.004805587697774172, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00482, "step": 173, "tokens/total": 5610024, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 79059} +{"epoch": 0.6796875, "grad_norm": 1.4421262741088867, "learning_rate": 8.100214524900103e-05, "loss": 0.007902286015450954, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00793, "step": 174, "tokens/total": 5643184, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 79498} +{"epoch": 0.68359375, "grad_norm": 0.16329306364059448, "learning_rate": 8.076468193435301e-05, "loss": 0.0010412737028673291, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00104, "step": 175, "tokens/total": 5672824, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 79922} +{"epoch": 0.6875, "grad_norm": 0.6699113249778748, "learning_rate": 8.052614644612253e-05, "loss": 0.0029319950845092535, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00294, "step": 176, "tokens/total": 5705576, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 80383} +{"epoch": 0.69140625, "grad_norm": 0.06702185422182083, "learning_rate": 8.028654871074489e-05, "loss": 0.0003368477337062359, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00034, "step": 177, "tokens/total": 5737840, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 80824} +{"epoch": 0.6953125, "grad_norm": 0.044816065579652786, "learning_rate": 8.004589869885986e-05, "loss": 0.00026682220050133765, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00027, "step": 178, "tokens/total": 5767552, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 81243} +{"epoch": 0.69921875, "grad_norm": 1.2610775232315063, "learning_rate": 7.980420642489674e-05, "loss": 0.011610487475991249, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01168, "step": 179, "tokens/total": 5802512, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 81716} +{"epoch": 0.703125, "grad_norm": 0.7388120293617249, "learning_rate": 7.95614819466576e-05, "loss": 0.0025765574537217617, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00258, "step": 180, "tokens/total": 5834984, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 82181} +{"epoch": 0.70703125, "grad_norm": 0.042610444128513336, "learning_rate": 7.931773536489872e-05, "loss": 7.307275518542156e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00007, "step": 181, "tokens/total": 5867336, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 82626} +{"epoch": 0.7109375, "grad_norm": 1.0904520750045776, "learning_rate": 7.907297682291035e-05, "loss": 0.017555490136146545, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01771, "step": 182, "tokens/total": 5899232, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 83089} +{"epoch": 0.71484375, "grad_norm": 2.03102707862854, "learning_rate": 7.882721650609442e-05, "loss": 0.0106183597818017, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.01067, "step": 183, "tokens/total": 5934816, "tokens/train_per_sec_per_gpu": 6.73, "tokens/trainable": 83590} +{"epoch": 0.71875, "grad_norm": 0.37600117921829224, "learning_rate": 7.85804646415409e-05, "loss": 0.004075631033629179, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.00408, "step": 184, "tokens/total": 5968120, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 84046} +{"epoch": 0.72265625, "grad_norm": 0.4327266812324524, "learning_rate": 7.833273149760207e-05, "loss": 0.006051763892173767, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00607, "step": 185, "tokens/total": 5998600, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 84517} +{"epoch": 0.7265625, "grad_norm": 2.2282471656799316, "learning_rate": 7.808402738346527e-05, "loss": 0.016216427087783813, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01635, "step": 186, "tokens/total": 6033832, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 84974} +{"epoch": 0.73046875, "grad_norm": 0.06660744547843933, "learning_rate": 7.783436264872382e-05, "loss": 0.0005763238295912743, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00058, "step": 187, "tokens/total": 6068192, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 85460} +{"epoch": 0.734375, "grad_norm": 0.9499677419662476, "learning_rate": 7.758374768294647e-05, "loss": 0.0029683164320886135, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00297, "step": 188, "tokens/total": 6101464, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 85939} +{"epoch": 0.73828125, "grad_norm": 0.42081552743911743, "learning_rate": 7.733219291524489e-05, "loss": 0.008591750636696815, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00863, "step": 189, "tokens/total": 6133008, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 86377} +{"epoch": 0.7421875, "grad_norm": 0.2749069035053253, "learning_rate": 7.707970881383977e-05, "loss": 0.006198354065418243, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.00622, "step": 190, "tokens/total": 6163048, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 86834} +{"epoch": 0.74609375, "grad_norm": 0.41557347774505615, "learning_rate": 7.682630588562518e-05, "loss": 0.0011461745016276836, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00115, "step": 191, "tokens/total": 6196072, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 87265} +{"epoch": 0.75, "grad_norm": 0.3493348956108093, "learning_rate": 7.657199467573129e-05, "loss": 0.00516095757484436, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00517, "step": 192, "tokens/total": 6227312, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 87747} +{"epoch": 0.75390625, "grad_norm": 0.30505824089050293, "learning_rate": 7.631678576708561e-05, "loss": 0.0030087886843830347, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00301, "step": 193, "tokens/total": 6260240, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 88239} +{"epoch": 0.7578125, "grad_norm": 0.07805999368429184, "learning_rate": 7.606068977997255e-05, "loss": 0.0004954148316755891, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.0005, "step": 194, "tokens/total": 6294280, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 88718} +{"epoch": 0.76171875, "grad_norm": 0.8711969256401062, "learning_rate": 7.580371737159148e-05, "loss": 0.006917244754731655, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00694, "step": 195, "tokens/total": 6323368, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 89129} +{"epoch": 0.765625, "grad_norm": 0.08992184698581696, "learning_rate": 7.554587923561324e-05, "loss": 0.0004628734022844583, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00046, "step": 196, "tokens/total": 6354960, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 89567} +{"epoch": 0.76953125, "grad_norm": 0.16043980419635773, "learning_rate": 7.528718610173511e-05, "loss": 0.000771163497120142, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.00077, "step": 197, "tokens/total": 6387208, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 89988} +{"epoch": 0.7734375, "grad_norm": 0.010454738512635231, "learning_rate": 7.502764873523431e-05, "loss": 8.462134428555146e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00008, "step": 198, "tokens/total": 6418720, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 90434} +{"epoch": 0.77734375, "grad_norm": 0.7501993179321289, "learning_rate": 7.476727793652011e-05, "loss": 0.02789936773478985, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.02829, "step": 199, "tokens/total": 6449184, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 90913} +{"epoch": 0.78125, "grad_norm": 0.7312188744544983, "learning_rate": 7.450608454068415e-05, "loss": 0.009919527918100357, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00997, "step": 200, "tokens/total": 6481824, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 91355} +{"epoch": 0.78515625, "grad_norm": 0.3267847001552582, "learning_rate": 7.424407941704987e-05, "loss": 0.00910334661602974, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00914, "step": 201, "tokens/total": 6515896, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 91856} +{"epoch": 0.7890625, "grad_norm": 0.04229301959276199, "learning_rate": 7.398127346871986e-05, "loss": 0.0003965634386986494, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.0004, "step": 202, "tokens/total": 6548784, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 92311} +{"epoch": 0.79296875, "grad_norm": 0.018465733155608177, "learning_rate": 7.371767763212238e-05, "loss": 0.0002634293050505221, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00026, "step": 203, "tokens/total": 6580256, "tokens/train_per_sec_per_gpu": 10.06, "tokens/trainable": 92764} +{"epoch": 0.796875, "grad_norm": 0.7805087566375732, "learning_rate": 7.345330287655617e-05, "loss": 0.004285633563995361, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00429, "step": 204, "tokens/total": 6613976, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 93227} +{"epoch": 0.80078125, "grad_norm": 0.2401893138885498, "learning_rate": 7.31881602037339e-05, "loss": 0.002241286914795637, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.00224, "step": 205, "tokens/total": 6644552, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 93675} +{"epoch": 0.8046875, "grad_norm": 0.22748994827270508, "learning_rate": 7.29222606473245e-05, "loss": 0.003245249157771468, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00325, "step": 206, "tokens/total": 6675056, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 94120} +{"epoch": 0.80859375, "grad_norm": 0.10334953665733337, "learning_rate": 7.265561527249383e-05, "loss": 0.0009179697372019291, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00092, "step": 207, "tokens/total": 6706624, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 94557} +{"epoch": 0.8125, "grad_norm": 0.6100345849990845, "learning_rate": 7.238823517544436e-05, "loss": 0.005382989998906851, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.0054, "step": 208, "tokens/total": 6741008, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 95035} +{"epoch": 0.81640625, "grad_norm": 0.5989669561386108, "learning_rate": 7.212013148295333e-05, "loss": 0.008767460472881794, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00881, "step": 209, "tokens/total": 6774872, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 95517} +{"epoch": 0.8203125, "grad_norm": 0.2534485459327698, "learning_rate": 7.185131535190975e-05, "loss": 0.0029698393773287535, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00297, "step": 210, "tokens/total": 6805816, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 96026} +{"epoch": 0.82421875, "grad_norm": 0.3060764670372009, "learning_rate": 7.158179796885005e-05, "loss": 0.0008332564029842615, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00083, "step": 211, "tokens/total": 6837592, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 96477} +{"epoch": 0.828125, "grad_norm": 0.06377515196800232, "learning_rate": 7.131159054949273e-05, "loss": 0.0011754062725231051, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00118, "step": 212, "tokens/total": 6874384, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 96951} +{"epoch": 0.83203125, "grad_norm": 0.012191381305456161, "learning_rate": 7.104070433827139e-05, "loss": 0.00016749955830164254, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00017, "step": 213, "tokens/total": 6905872, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 97350} +{"epoch": 0.8359375, "grad_norm": 0.017519162967801094, "learning_rate": 7.076915060786705e-05, "loss": 0.0002556497056502849, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.00026, "step": 214, "tokens/total": 6937984, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 97792} +{"epoch": 0.83984375, "grad_norm": 0.3223553001880646, "learning_rate": 7.049694065873882e-05, "loss": 0.002114279195666313, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00212, "step": 215, "tokens/total": 6970704, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 98257} +{"epoch": 0.84375, "grad_norm": 0.07253021746873856, "learning_rate": 7.022408581865382e-05, "loss": 0.0008622311288490891, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00086, "step": 216, "tokens/total": 7004616, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 98731} +{"epoch": 0.84765625, "grad_norm": 0.2613694369792938, "learning_rate": 6.99505974422157e-05, "loss": 0.0010344523470848799, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00103, "step": 217, "tokens/total": 7036880, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 99212} +{"epoch": 0.8515625, "grad_norm": 0.9310376644134521, "learning_rate": 6.967648691039213e-05, "loss": 0.0030194250866770744, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00302, "step": 218, "tokens/total": 7068208, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 99654} +{"epoch": 0.85546875, "grad_norm": 0.014167922548949718, "learning_rate": 6.940176563004123e-05, "loss": 0.00017240707529708743, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00017, "step": 219, "tokens/total": 7098112, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 100086} +{"epoch": 0.859375, "grad_norm": 0.3772371709346771, "learning_rate": 6.912644503343682e-05, "loss": 0.0014148122863844037, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00142, "step": 220, "tokens/total": 7132432, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 100524} +{"epoch": 0.86328125, "grad_norm": 0.2959020137786865, "learning_rate": 6.885053657779273e-05, "loss": 0.0018009706400334835, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.0018, "step": 221, "tokens/total": 7164016, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 100996} +{"epoch": 0.8671875, "grad_norm": 0.27141451835632324, "learning_rate": 6.857405174478604e-05, "loss": 0.0012848550686612725, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00129, "step": 222, "tokens/total": 7198344, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 101449} +{"epoch": 0.87109375, "grad_norm": 0.004032483324408531, "learning_rate": 6.82970020400792e-05, "loss": 4.968723442289047e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00005, "step": 223, "tokens/total": 7232192, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 101905} +{"epoch": 0.875, "grad_norm": 0.15584298968315125, "learning_rate": 6.801939899284132e-05, "loss": 0.0005682804621756077, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00057, "step": 224, "tokens/total": 7266984, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 102411} +{"epoch": 0.87890625, "grad_norm": 0.009436636231839657, "learning_rate": 6.774125415526827e-05, "loss": 8.2856829976663e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00008, "step": 225, "tokens/total": 7299512, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 102882} +{"epoch": 0.8828125, "grad_norm": 0.004759322386234999, "learning_rate": 6.746257910210214e-05, "loss": 1.891222200356424e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00002, "step": 226, "tokens/total": 7331120, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 103332} +{"epoch": 0.88671875, "grad_norm": 0.9573491215705872, "learning_rate": 6.718338543014937e-05, "loss": 0.0020035577472299337, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00201, "step": 227, "tokens/total": 7360504, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 103790} +{"epoch": 0.890625, "grad_norm": 0.28103822469711304, "learning_rate": 6.69036847577983e-05, "loss": 0.0018197105964645743, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00182, "step": 228, "tokens/total": 7390760, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 104246} +{"epoch": 0.89453125, "grad_norm": 0.4739796817302704, "learning_rate": 6.662348872453553e-05, "loss": 0.006930890958756208, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00695, "step": 229, "tokens/total": 7423352, "tokens/train_per_sec_per_gpu": 10.5, "tokens/trainable": 104707} +{"epoch": 0.8984375, "grad_norm": 0.10424702614545822, "learning_rate": 6.63428089904618e-05, "loss": 0.00047341029858216643, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00047, "step": 230, "tokens/total": 7457152, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 105167} +{"epoch": 0.90234375, "grad_norm": 0.0532701350748539, "learning_rate": 6.60616572358065e-05, "loss": 0.00011039472883567214, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00011, "step": 231, "tokens/total": 7488648, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 105576} +{"epoch": 0.90625, "grad_norm": 0.0597260519862175, "learning_rate": 6.578004516044172e-05, "loss": 0.00022284116130322218, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00022, "step": 232, "tokens/total": 7522160, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 106043} +{"epoch": 0.91015625, "grad_norm": 0.0028958264738321304, "learning_rate": 6.549798448339548e-05, "loss": 1.804477142286487e-05, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00002, "step": 233, "tokens/total": 7557336, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 106506} +{"epoch": 0.9140625, "grad_norm": 0.0841817557811737, "learning_rate": 6.521548694236384e-05, "loss": 0.00018528164946474135, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00019, "step": 234, "tokens/total": 7590432, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 106951} +{"epoch": 0.91796875, "grad_norm": 5.438999176025391, "learning_rate": 6.493256429322259e-05, "loss": 0.039688821882009506, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.04049, "step": 235, "tokens/total": 7623240, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 107382} +{"epoch": 0.921875, "grad_norm": 1.075016736984253, "learning_rate": 6.464922830953799e-05, "loss": 0.008009557612240314, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00804, "step": 236, "tokens/total": 7657968, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 107814} +{"epoch": 0.92578125, "grad_norm": 0.05186557024717331, "learning_rate": 6.436549078207688e-05, "loss": 0.00033610209356993437, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00034, "step": 237, "tokens/total": 7689320, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 108274} +{"epoch": 0.9296875, "grad_norm": 2.8328685760498047, "learning_rate": 6.408136351831592e-05, "loss": 0.0645897313952446, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.06672, "step": 238, "tokens/total": 7721568, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 108714} +{"epoch": 0.93359375, "grad_norm": 0.5768492817878723, "learning_rate": 6.379685834195036e-05, "loss": 0.0017036491772159934, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00171, "step": 239, "tokens/total": 7756464, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 109220} +{"epoch": 0.9375, "grad_norm": 0.6531472206115723, "learning_rate": 6.351198709240186e-05, "loss": 0.004286657087504864, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.0043, "step": 240, "tokens/total": 7791616, "tokens/train_per_sec_per_gpu": 7.01, "tokens/trainable": 109672} +{"epoch": 0.94140625, "grad_norm": 0.24345603585243225, "learning_rate": 6.32267616243259e-05, "loss": 0.005883925594389439, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.0059, "step": 241, "tokens/total": 7823520, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 110135} +{"epoch": 0.9453125, "grad_norm": 0.20511305332183838, "learning_rate": 6.294119380711849e-05, "loss": 0.001242971047759056, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00124, "step": 242, "tokens/total": 7854424, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 110563} +{"epoch": 0.94921875, "grad_norm": 0.7665442824363708, "learning_rate": 6.265529552442209e-05, "loss": 0.011101300828158855, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01116, "step": 243, "tokens/total": 7887160, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 111049} +{"epoch": 0.953125, "grad_norm": 0.3372010886669159, "learning_rate": 6.236907867363127e-05, "loss": 0.002675319090485573, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00268, "step": 244, "tokens/total": 7918736, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 111495} +{"epoch": 0.95703125, "grad_norm": 0.326371431350708, "learning_rate": 6.208255516539749e-05, "loss": 0.0023236856795847416, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00233, "step": 245, "tokens/total": 7949448, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 111968} +{"epoch": 0.9609375, "grad_norm": 0.24030368030071259, "learning_rate": 6.179573692313344e-05, "loss": 0.001217978773638606, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00122, "step": 246, "tokens/total": 7984680, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 112416} +{"epoch": 0.96484375, "grad_norm": 0.6710991263389587, "learning_rate": 6.150863588251694e-05, "loss": 0.008254293352365494, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00829, "step": 247, "tokens/total": 8019816, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 112882} +{"epoch": 0.96875, "grad_norm": 0.49263232946395874, "learning_rate": 6.122126399099419e-05, "loss": 0.006979962810873985, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.007, "step": 248, "tokens/total": 8056272, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 113390} +{"epoch": 0.97265625, "grad_norm": 1.1866216659545898, "learning_rate": 6.0933633207282615e-05, "loss": 0.014414223842322826, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.01452, "step": 249, "tokens/total": 8090224, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 113904} +{"epoch": 0.9765625, "grad_norm": 0.6176316142082214, "learning_rate": 6.064575550087316e-05, "loss": 0.008024574257433414, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00806, "step": 250, "tokens/total": 8124528, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 114351} +{"epoch": 0.98046875, "grad_norm": 0.25469398498535156, "learning_rate": 6.0357642851532245e-05, "loss": 0.00273851677775383, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00274, "step": 251, "tokens/total": 8159688, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 114842} +{"epoch": 0.984375, "grad_norm": 0.1834985315799713, "learning_rate": 6.0069307248803294e-05, "loss": 0.0009540771134197712, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00095, "step": 252, "tokens/total": 8191800, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 115263} +{"epoch": 0.98828125, "grad_norm": 0.2972082197666168, "learning_rate": 5.9780760691507635e-05, "loss": 0.000961610465310514, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00096, "step": 253, "tokens/total": 8221752, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 115703} +{"epoch": 0.9921875, "grad_norm": 0.10320135205984116, "learning_rate": 5.9492015187245334e-05, "loss": 0.0003912528045475483, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00039, "step": 254, "tokens/total": 8255648, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 116157} +{"epoch": 0.99609375, "grad_norm": 19.270978927612305, "learning_rate": 5.920308275189541e-05, "loss": 0.006222342606633902, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00624, "step": 255, "tokens/total": 8286664, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 116567} +{"epoch": 1.0, "grad_norm": 0.4240311086177826, "learning_rate": 5.8913975409115874e-05, "loss": 0.003089963924139738, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00309, "step": 256, "tokens/total": 8318368, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 117030} +{"epoch": 1.00390625, "grad_norm": 2.7514398097991943, "learning_rate": 5.8624705189843395e-05, "loss": 0.006667394656687975, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00669, "step": 257, "tokens/total": 8353336, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 117517} +{"epoch": 1.0078125, "grad_norm": 0.19045251607894897, "learning_rate": 5.833528413179249e-05, "loss": 0.0008756911847740412, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00088, "step": 258, "tokens/total": 8389112, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 118015} +{"epoch": 1.01171875, "grad_norm": 1.7768446207046509, "learning_rate": 5.80457242789548e-05, "loss": 0.004494940396398306, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00451, "step": 259, "tokens/total": 8426000, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 118520} +{"epoch": 1.015625, "grad_norm": 0.12111207842826843, "learning_rate": 5.77560376810977e-05, "loss": 0.0006445335457101464, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.00064, "step": 260, "tokens/total": 8458568, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 118992} +{"epoch": 1.01953125, "grad_norm": 0.01776093989610672, "learning_rate": 5.7466236393263005e-05, "loss": 0.00016423288616351783, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00016, "step": 261, "tokens/total": 8490840, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 119438} +{"epoch": 1.0234375, "grad_norm": 0.17523367702960968, "learning_rate": 5.717633247526522e-05, "loss": 0.0006255090120248497, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00063, "step": 262, "tokens/total": 8521792, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 119881} +{"epoch": 1.02734375, "grad_norm": 0.460614949464798, "learning_rate": 5.688633799118971e-05, "loss": 0.004987872205674648, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.005, "step": 263, "tokens/total": 8551336, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 120285} +{"epoch": 1.03125, "grad_norm": 2.2836837768554688, "learning_rate": 5.659626500889066e-05, "loss": 0.0039212824776768684, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00393, "step": 264, "tokens/total": 8579872, "tokens/train_per_sec_per_gpu": 7.04, "tokens/trainable": 120755} +{"epoch": 1.03515625, "grad_norm": 5.9923295974731445, "learning_rate": 5.6306125599488905e-05, "loss": 0.001277842791751027, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00128, "step": 265, "tokens/total": 8608920, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 121194} +{"epoch": 1.0390625, "grad_norm": 0.3126618266105652, "learning_rate": 5.601593183686955e-05, "loss": 0.001982432557269931, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00198, "step": 266, "tokens/total": 8643600, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 121670} +{"epoch": 1.04296875, "grad_norm": 0.743201732635498, "learning_rate": 5.572569579717961e-05, "loss": 0.008599260821938515, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00864, "step": 267, "tokens/total": 8676032, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 122142} +{"epoch": 1.046875, "grad_norm": 0.0287510696798563, "learning_rate": 5.543542955832538e-05, "loss": 0.00022218791127670556, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00022, "step": 268, "tokens/total": 8713240, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 122585} +{"epoch": 1.05078125, "grad_norm": 4.626601219177246, "learning_rate": 5.514514519946986e-05, "loss": 0.01276959478855133, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01285, "step": 269, "tokens/total": 8751664, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 123091} +{"epoch": 1.0546875, "grad_norm": 0.15515683591365814, "learning_rate": 5.485485480053015e-05, "loss": 0.0005239187739789486, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00052, "step": 270, "tokens/total": 8786112, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 123505} +{"epoch": 1.05859375, "grad_norm": 0.10346806794404984, "learning_rate": 5.4564570441674645e-05, "loss": 0.00032699445728212595, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00033, "step": 271, "tokens/total": 8816920, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 123953} +{"epoch": 1.0625, "grad_norm": 0.16481345891952515, "learning_rate": 5.42743042028204e-05, "loss": 0.0007148925215005875, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00072, "step": 272, "tokens/total": 8851408, "tokens/train_per_sec_per_gpu": 6.8, "tokens/trainable": 124400} +{"epoch": 1.06640625, "grad_norm": 0.383842796087265, "learning_rate": 5.3984068163130464e-05, "loss": 0.0017706684302538633, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00177, "step": 273, "tokens/total": 8883768, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 124870} +{"epoch": 1.0703125, "grad_norm": 0.19806760549545288, "learning_rate": 5.369387440051111e-05, "loss": 9.398221300216392e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00009, "step": 274, "tokens/total": 8916200, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 125333} +{"epoch": 1.07421875, "grad_norm": 0.009305611252784729, "learning_rate": 5.340373499110935e-05, "loss": 7.427335367538035e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00007, "step": 275, "tokens/total": 8949920, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 125834} +{"epoch": 1.078125, "grad_norm": 0.030878497287631035, "learning_rate": 5.3113662008810304e-05, "loss": 0.00016023176431190223, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00016, "step": 276, "tokens/total": 8986448, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 126316} +{"epoch": 1.08203125, "grad_norm": 0.15672709047794342, "learning_rate": 5.282366752473479e-05, "loss": 0.0010859487811103463, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00109, "step": 277, "tokens/total": 9020600, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 126798} +{"epoch": 1.0859375, "grad_norm": 0.20113767683506012, "learning_rate": 5.2533763606737005e-05, "loss": 0.0008686523069627583, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00087, "step": 278, "tokens/total": 9056040, "tokens/train_per_sec_per_gpu": 6.87, "tokens/trainable": 127223} +{"epoch": 1.08984375, "grad_norm": 0.45938169956207275, "learning_rate": 5.224396231890232e-05, "loss": 0.00449849758297205, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00451, "step": 279, "tokens/total": 9087072, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 127672} +{"epoch": 1.09375, "grad_norm": 0.12441904097795486, "learning_rate": 5.195427572104522e-05, "loss": 0.001518811797723174, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00152, "step": 280, "tokens/total": 9118376, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 128116} +{"epoch": 1.09765625, "grad_norm": 0.2862607538700104, "learning_rate": 5.166471586820751e-05, "loss": 0.005550042726099491, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00557, "step": 281, "tokens/total": 9152704, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 128589} +{"epoch": 1.1015625, "grad_norm": 0.010912097059190273, "learning_rate": 5.1375294810156615e-05, "loss": 8.855060150381178e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00009, "step": 282, "tokens/total": 9184448, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 129036} +{"epoch": 1.10546875, "grad_norm": 0.5771896243095398, "learning_rate": 5.1086024590884144e-05, "loss": 0.005922919139266014, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00594, "step": 283, "tokens/total": 9217392, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 129485} +{"epoch": 1.109375, "grad_norm": 0.17946167290210724, "learning_rate": 5.079691724810461e-05, "loss": 0.0002871537581086159, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00029, "step": 284, "tokens/total": 9252016, "tokens/train_per_sec_per_gpu": 7.08, "tokens/trainable": 129920} +{"epoch": 1.11328125, "grad_norm": 0.23658312857151031, "learning_rate": 5.0507984812754684e-05, "loss": 0.0033509586937725544, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00336, "step": 285, "tokens/total": 9284008, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 130417} +{"epoch": 1.1171875, "grad_norm": 0.04220741614699364, "learning_rate": 5.021923930849237e-05, "loss": 0.00025076151359826326, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00025, "step": 286, "tokens/total": 9315984, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 130903} +{"epoch": 1.12109375, "grad_norm": 0.02998567372560501, "learning_rate": 4.99306927511967e-05, "loss": 0.00011217327846679837, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00011, "step": 287, "tokens/total": 9347376, "tokens/train_per_sec_per_gpu": 10.41, "tokens/trainable": 131343} +{"epoch": 1.125, "grad_norm": 0.009595762938261032, "learning_rate": 4.964235714846775e-05, "loss": 4.6648889110656455e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00005, "step": 288, "tokens/total": 9377424, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 131763} +{"epoch": 1.12890625, "grad_norm": 0.0029994689393788576, "learning_rate": 4.9354244499126866e-05, "loss": 2.9481172532541677e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00003, "step": 289, "tokens/total": 9408104, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 132229} +{"epoch": 1.1328125, "grad_norm": 0.10034480690956116, "learning_rate": 4.90663667927174e-05, "loss": 0.00047778760199435055, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00048, "step": 290, "tokens/total": 9439944, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 132678} +{"epoch": 1.13671875, "grad_norm": 0.07781729847192764, "learning_rate": 4.877873600900581e-05, "loss": 0.00048447042354382575, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00048, "step": 291, "tokens/total": 9472104, "tokens/train_per_sec_per_gpu": 6.22, "tokens/trainable": 133136} +{"epoch": 1.140625, "grad_norm": 0.35858193039894104, "learning_rate": 4.849136411748306e-05, "loss": 0.0031722101848572493, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00318, "step": 292, "tokens/total": 9508336, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 133608} +{"epoch": 1.14453125, "grad_norm": 0.4816785156726837, "learning_rate": 4.8204263076866574e-05, "loss": 0.007220402359962463, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00725, "step": 293, "tokens/total": 9541552, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 134108} +{"epoch": 1.1484375, "grad_norm": 0.0014753404539078474, "learning_rate": 4.791744483460251e-05, "loss": 1.7065851352526806e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00002, "step": 294, "tokens/total": 9575072, "tokens/train_per_sec_per_gpu": 6.57, "tokens/trainable": 134541} +{"epoch": 1.15234375, "grad_norm": 0.28012168407440186, "learning_rate": 4.7630921326368736e-05, "loss": 0.002125541912391782, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00213, "step": 295, "tokens/total": 9610344, "tokens/train_per_sec_per_gpu": 6.4, "tokens/trainable": 134966} +{"epoch": 1.15625, "grad_norm": 0.33942702412605286, "learning_rate": 4.7344704475577916e-05, "loss": 0.0009926467901095748, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00099, "step": 296, "tokens/total": 9639728, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 135402} +{"epoch": 1.16015625, "grad_norm": 0.35907575488090515, "learning_rate": 4.705880619288153e-05, "loss": 0.0018794597126543522, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00188, "step": 297, "tokens/total": 9671424, "tokens/train_per_sec_per_gpu": 7.09, "tokens/trainable": 135804} +{"epoch": 1.1640625, "grad_norm": 0.1625092625617981, "learning_rate": 4.677323837567412e-05, "loss": 0.001350727048702538, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00135, "step": 298, "tokens/total": 9700808, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 136231} +{"epoch": 1.16796875, "grad_norm": 0.042915310710668564, "learning_rate": 4.6488012907598146e-05, "loss": 0.00016318981943186373, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00016, "step": 299, "tokens/total": 9732312, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 136705} +{"epoch": 1.171875, "grad_norm": 1.6889539957046509, "learning_rate": 4.620314165804964e-05, "loss": 0.008080918341875076, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00811, "step": 300, "tokens/total": 9764624, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 137178} +{"epoch": 1.17578125, "grad_norm": 0.4560539126396179, "learning_rate": 4.591863648168407e-05, "loss": 0.0036483732983469963, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00366, "step": 301, "tokens/total": 9798368, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 137643} +{"epoch": 1.1796875, "grad_norm": 0.028114963322877884, "learning_rate": 4.5634509217923135e-05, "loss": 0.0001654226944083348, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00017, "step": 302, "tokens/total": 9830192, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 138078} +{"epoch": 1.18359375, "grad_norm": 0.0806829109787941, "learning_rate": 4.535077169046201e-05, "loss": 0.000467342440970242, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00047, "step": 303, "tokens/total": 9863112, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 138515} +{"epoch": 1.1875, "grad_norm": 0.4055023789405823, "learning_rate": 4.506743570677743e-05, "loss": 0.003944179974496365, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00395, "step": 304, "tokens/total": 9893192, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 138948} +{"epoch": 1.19140625, "grad_norm": 0.042137522250413895, "learning_rate": 4.478451305763618e-05, "loss": 0.0003698138170875609, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00037, "step": 305, "tokens/total": 9925712, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 139408} +{"epoch": 1.1953125, "grad_norm": 0.007556610740721226, "learning_rate": 4.450201551660454e-05, "loss": 2.7271618819213472e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00003, "step": 306, "tokens/total": 9955736, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 139840} +{"epoch": 1.19921875, "grad_norm": 0.40838924050331116, "learning_rate": 4.4219954839558276e-05, "loss": 0.005668125580996275, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00568, "step": 307, "tokens/total": 9987160, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 140281} +{"epoch": 1.203125, "grad_norm": 2.4760656356811523, "learning_rate": 4.393834276419352e-05, "loss": 0.0010095905745401978, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00101, "step": 308, "tokens/total": 10017360, "tokens/train_per_sec_per_gpu": 11.13, "tokens/trainable": 140776} +{"epoch": 1.20703125, "grad_norm": 0.006324793212115765, "learning_rate": 4.36571910095382e-05, "loss": 2.0802173821721226e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00002, "step": 309, "tokens/total": 10052056, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 141228} +{"epoch": 1.2109375, "grad_norm": 0.006749860942363739, "learning_rate": 4.337651127546448e-05, "loss": 5.704734576283954e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00006, "step": 310, "tokens/total": 10085424, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 141679} +{"epoch": 1.21484375, "grad_norm": 0.29972073435783386, "learning_rate": 4.3096315242201736e-05, "loss": 0.002209673635661602, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00221, "step": 311, "tokens/total": 10116824, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 142122} +{"epoch": 1.21875, "grad_norm": 0.17902274429798126, "learning_rate": 4.2816614569850635e-05, "loss": 0.0008353852899745107, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00084, "step": 312, "tokens/total": 10150376, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 142561} +{"epoch": 1.22265625, "grad_norm": 0.23005418479442596, "learning_rate": 4.2537420897897864e-05, "loss": 0.0005114163504913449, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00051, "step": 313, "tokens/total": 10184432, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 143046} +{"epoch": 1.2265625, "grad_norm": 0.18722085654735565, "learning_rate": 4.225874584473174e-05, "loss": 0.0007510854629799724, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00075, "step": 314, "tokens/total": 10214848, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 143493} +{"epoch": 1.23046875, "grad_norm": 0.08282724022865295, "learning_rate": 4.19806010071587e-05, "loss": 0.0008623105240985751, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00086, "step": 315, "tokens/total": 10247952, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 143956} +{"epoch": 1.234375, "grad_norm": 0.11827684938907623, "learning_rate": 4.170299795992081e-05, "loss": 0.0007219837279990315, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00072, "step": 316, "tokens/total": 10280648, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 144411} +{"epoch": 1.23828125, "grad_norm": 0.036739103496074677, "learning_rate": 4.142594825521398e-05, "loss": 0.0002865194692276418, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00029, "step": 317, "tokens/total": 10317712, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 144892} +{"epoch": 1.2421875, "grad_norm": 0.003821773687377572, "learning_rate": 4.114946342220728e-05, "loss": 3.361784911248833e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00003, "step": 318, "tokens/total": 10351632, "tokens/train_per_sec_per_gpu": 6.97, "tokens/trainable": 145339} +{"epoch": 1.24609375, "grad_norm": 0.19647598266601562, "learning_rate": 4.087355496656321e-05, "loss": 0.001789616420865059, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00179, "step": 319, "tokens/total": 10383336, "tokens/train_per_sec_per_gpu": 10.15, "tokens/trainable": 145811} +{"epoch": 1.25, "grad_norm": 0.02719424106180668, "learning_rate": 4.05982343699588e-05, "loss": 0.00016090870485641062, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00016, "step": 320, "tokens/total": 10418016, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 146314} +{"epoch": 1.25390625, "grad_norm": 0.3963592052459717, "learning_rate": 4.0323513089607876e-05, "loss": 0.0007934526074677706, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00079, "step": 321, "tokens/total": 10451600, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 146781} +{"epoch": 1.2578125, "grad_norm": 0.016152333468198776, "learning_rate": 4.004940255778431e-05, "loss": 7.413173443637788e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00007, "step": 322, "tokens/total": 10487568, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 147225} +{"epoch": 1.26171875, "grad_norm": 0.2671172618865967, "learning_rate": 3.977591418134619e-05, "loss": 0.002280148211866617, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00228, "step": 323, "tokens/total": 10520976, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 147673} +{"epoch": 1.265625, "grad_norm": 0.03466499224305153, "learning_rate": 3.95030593412612e-05, "loss": 0.00019890772819053382, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.0002, "step": 324, "tokens/total": 10552776, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 148103} +{"epoch": 1.26953125, "grad_norm": 0.002327781170606613, "learning_rate": 3.923084939213296e-05, "loss": 1.2974270248378161e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00001, "step": 325, "tokens/total": 10582576, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 148535} +{"epoch": 1.2734375, "grad_norm": 0.07367193698883057, "learning_rate": 3.895929566172861e-05, "loss": 0.00021919523715041578, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00022, "step": 326, "tokens/total": 10615600, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 148999} +{"epoch": 1.27734375, "grad_norm": 0.0270446315407753, "learning_rate": 3.868840945050728e-05, "loss": 0.00020972943457309157, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00021, "step": 327, "tokens/total": 10650688, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 149431} +{"epoch": 1.28125, "grad_norm": 0.03800337761640549, "learning_rate": 3.841820203114995e-05, "loss": 0.00014955131337046623, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00015, "step": 328, "tokens/total": 10681816, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 149886} +{"epoch": 1.28515625, "grad_norm": 0.007131698075681925, "learning_rate": 3.814868464809027e-05, "loss": 5.3322721214499325e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00005, "step": 329, "tokens/total": 10711728, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 150288} +{"epoch": 1.2890625, "grad_norm": 0.9082645773887634, "learning_rate": 3.787986851704667e-05, "loss": 0.0032864573877304792, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00329, "step": 330, "tokens/total": 10744952, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 150733} +{"epoch": 1.29296875, "grad_norm": 0.0012546086218208075, "learning_rate": 3.7611764824555654e-05, "loss": 4.211323812342016e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.0, "step": 331, "tokens/total": 10776192, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 151207} +{"epoch": 1.296875, "grad_norm": 0.003985502757132053, "learning_rate": 3.734438472750619e-05, "loss": 1.5108989828149788e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00002, "step": 332, "tokens/total": 10813336, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 151694} +{"epoch": 1.30078125, "grad_norm": 0.006909841205924749, "learning_rate": 3.707773935267552e-05, "loss": 1.7819931599660777e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00002, "step": 333, "tokens/total": 10847808, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 152188} +{"epoch": 1.3046875, "grad_norm": 0.008703690022230148, "learning_rate": 3.68118397962661e-05, "loss": 2.042602136498317e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00002, "step": 334, "tokens/total": 10879896, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 152596} +{"epoch": 1.30859375, "grad_norm": 0.17412595450878143, "learning_rate": 3.654669712344384e-05, "loss": 0.0003115733852609992, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00031, "step": 335, "tokens/total": 10908976, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 153052} +{"epoch": 1.3125, "grad_norm": 0.009348472580313683, "learning_rate": 3.628232236787763e-05, "loss": 3.759480023290962e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00004, "step": 336, "tokens/total": 10943152, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 153491} +{"epoch": 1.31640625, "grad_norm": 0.07005197554826736, "learning_rate": 3.6018726531280144e-05, "loss": 0.00020585214952006936, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00021, "step": 337, "tokens/total": 10978184, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 153983} +{"epoch": 1.3203125, "grad_norm": 0.0004969299770891666, "learning_rate": 3.575592058295017e-05, "loss": 4.205669938528445e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.0, "step": 338, "tokens/total": 11010056, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 154464} +{"epoch": 1.32421875, "grad_norm": 0.00011099904804723337, "learning_rate": 3.549391545931585e-05, "loss": 1.4313791325548664e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.0, "step": 339, "tokens/total": 11040968, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 154924} +{"epoch": 1.328125, "grad_norm": 0.030560282990336418, "learning_rate": 3.5232722063479914e-05, "loss": 5.009127562516369e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00005, "step": 340, "tokens/total": 11073424, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 155373} +{"epoch": 1.33203125, "grad_norm": 0.00793465320020914, "learning_rate": 3.49723512647657e-05, "loss": 2.6412808438180946e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00003, "step": 341, "tokens/total": 11109224, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 155873} +{"epoch": 1.3359375, "grad_norm": 0.0004021470667794347, "learning_rate": 3.471281389826491e-05, "loss": 4.265066309017129e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.0, "step": 342, "tokens/total": 11139856, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 156278} +{"epoch": 1.33984375, "grad_norm": 0.0017199970316141844, "learning_rate": 3.4454120764386764e-05, "loss": 8.066249392868485e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00001, "step": 343, "tokens/total": 11175224, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 156733} +{"epoch": 1.34375, "grad_norm": 0.35497182607650757, "learning_rate": 3.4196282628408526e-05, "loss": 0.0019982450176030397, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.002, "step": 344, "tokens/total": 11207112, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 157203} +{"epoch": 1.34765625, "grad_norm": 0.5666045546531677, "learning_rate": 3.3939310220027456e-05, "loss": 0.001291808788664639, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00129, "step": 345, "tokens/total": 11240432, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 157707} +{"epoch": 1.3515625, "grad_norm": 0.001324445242062211, "learning_rate": 3.3683214232914404e-05, "loss": 4.537846962193726e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.0, "step": 346, "tokens/total": 11274584, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 158180} +{"epoch": 1.35546875, "grad_norm": 0.3400708734989166, "learning_rate": 3.342800532426873e-05, "loss": 0.001779093872755766, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00178, "step": 347, "tokens/total": 11307376, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 158646} +{"epoch": 1.359375, "grad_norm": 0.07066213339567184, "learning_rate": 3.317369411437484e-05, "loss": 0.00019483048527035862, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00019, "step": 348, "tokens/total": 11340640, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 159115} +{"epoch": 1.36328125, "grad_norm": 0.006788549944758415, "learning_rate": 3.292029118616024e-05, "loss": 3.840436693280935e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00004, "step": 349, "tokens/total": 11371632, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 159538} +{"epoch": 1.3671875, "grad_norm": 0.0025970651768147945, "learning_rate": 3.266780708475511e-05, "loss": 1.1095777153968811e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00001, "step": 350, "tokens/total": 11404248, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 160016} +{"epoch": 1.37109375, "grad_norm": 1.303378939628601, "learning_rate": 3.241625231705354e-05, "loss": 0.0009227151167578995, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00092, "step": 351, "tokens/total": 11439552, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 160475} +{"epoch": 1.375, "grad_norm": 0.00021055919933132827, "learning_rate": 3.216563735127618e-05, "loss": 2.054864580713911e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.0, "step": 352, "tokens/total": 11470944, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 160952} +{"epoch": 1.37890625, "grad_norm": 0.020578347146511078, "learning_rate": 3.191597261653475e-05, "loss": 6.776272493880242e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00007, "step": 353, "tokens/total": 11506304, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 161420} +{"epoch": 1.3828125, "grad_norm": 1.2227511405944824, "learning_rate": 3.166726850239794e-05, "loss": 0.012594059109687805, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01267, "step": 354, "tokens/total": 11535424, "tokens/train_per_sec_per_gpu": 10.98, "tokens/trainable": 161868} +{"epoch": 1.38671875, "grad_norm": 0.0010075237369164824, "learning_rate": 3.141953535845912e-05, "loss": 4.756328962685075e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.0, "step": 355, "tokens/total": 11565176, "tokens/train_per_sec_per_gpu": 10.25, "tokens/trainable": 162318} +{"epoch": 1.390625, "grad_norm": 0.017895415425300598, "learning_rate": 3.11727834939056e-05, "loss": 5.204259650781751e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00005, "step": 356, "tokens/total": 11595528, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 162789} +{"epoch": 1.39453125, "grad_norm": 0.01994381844997406, "learning_rate": 3.092702317708967e-05, "loss": 9.667977428762242e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.0001, "step": 357, "tokens/total": 11627248, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 163254} +{"epoch": 1.3984375, "grad_norm": 0.05982052534818649, "learning_rate": 3.0682264635101276e-05, "loss": 0.00011288843234069645, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00011, "step": 358, "tokens/total": 11659440, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 163715} +{"epoch": 1.40234375, "grad_norm": 0.04946693405508995, "learning_rate": 3.0438518053342407e-05, "loss": 0.00016261912242043763, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00016, "step": 359, "tokens/total": 11693288, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 164197} +{"epoch": 1.40625, "grad_norm": 0.1465839296579361, "learning_rate": 3.0195793575103266e-05, "loss": 0.0005365965771488845, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00054, "step": 360, "tokens/total": 11730888, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 164661} +{"epoch": 1.41015625, "grad_norm": 0.007158411666750908, "learning_rate": 2.9954101301140146e-05, "loss": 3.0362134566530585e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00003, "step": 361, "tokens/total": 11765656, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.5402820706367493, "learning_rate": 2.9713451289255123e-05, "loss": 0.0020574245136231184, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00206, "step": 362, "tokens/total": 11793456, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 165552} +{"epoch": 1.41796875, "grad_norm": 0.0024476938415318727, "learning_rate": 2.9473853553877484e-05, "loss": 7.664217264391482e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00001, "step": 363, "tokens/total": 11826280, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 166026} +{"epoch": 1.421875, "grad_norm": 0.6001955270767212, "learning_rate": 2.9235318065647e-05, "loss": 0.0007762835593894124, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00078, "step": 364, "tokens/total": 11863552, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 166486} +{"epoch": 1.42578125, "grad_norm": 0.0009058219729922712, "learning_rate": 2.8997854750998964e-05, "loss": 1.7165425560961012e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.0, "step": 365, "tokens/total": 11898616, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 166959} +{"epoch": 1.4296875, "grad_norm": 0.0002914570504799485, "learning_rate": 2.8761473491751258e-05, "loss": 2.291881401106366e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.0, "step": 366, "tokens/total": 11930728, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 167394} +{"epoch": 1.43359375, "grad_norm": 0.005134170409291983, "learning_rate": 2.8526184124692883e-05, "loss": 1.1531897143868264e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00001, "step": 367, "tokens/total": 11964976, "tokens/train_per_sec_per_gpu": 6.39, "tokens/trainable": 167832} +{"epoch": 1.4375, "grad_norm": 0.00719701312482357, "learning_rate": 2.829199644117484e-05, "loss": 2.7001791750080884e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00003, "step": 368, "tokens/total": 11996696, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 168242} +{"epoch": 1.44140625, "grad_norm": 0.014442838728427887, "learning_rate": 2.8058920186702553e-05, "loss": 4.3908788939006627e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00004, "step": 369, "tokens/total": 12033432, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 168715} +{"epoch": 1.4453125, "grad_norm": 0.0015256531769409776, "learning_rate": 2.782696506053033e-05, "loss": 6.504733391921036e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00001, "step": 370, "tokens/total": 12067616, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 169147} +{"epoch": 1.44921875, "grad_norm": 0.00020751018018927425, "learning_rate": 2.7596140715257824e-05, "loss": 3.0205178518372122e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.0, "step": 371, "tokens/total": 12099008, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 169628} +{"epoch": 1.453125, "grad_norm": 0.0108110336586833, "learning_rate": 2.7366456756428184e-05, "loss": 3.277149880887009e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00003, "step": 372, "tokens/total": 12131304, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 170085} +{"epoch": 1.45703125, "grad_norm": 0.0007074615568853915, "learning_rate": 2.7137922742128486e-05, "loss": 5.262364538793918e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00001, "step": 373, "tokens/total": 12164104, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 170584} +{"epoch": 1.4609375, "grad_norm": 0.7127290964126587, "learning_rate": 2.691054818259188e-05, "loss": 0.004926634952425957, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00494, "step": 374, "tokens/total": 12197776, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 171058} +{"epoch": 1.46484375, "grad_norm": 0.1583392471075058, "learning_rate": 2.6684342539801933e-05, "loss": 9.320479148300365e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00009, "step": 375, "tokens/total": 12228064, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 171518} +{"epoch": 1.46875, "grad_norm": 0.039220355451107025, "learning_rate": 2.645931522709877e-05, "loss": 0.00017089894390664995, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00017, "step": 376, "tokens/total": 12261688, "tokens/train_per_sec_per_gpu": 10.74, "tokens/trainable": 172007} +{"epoch": 1.47265625, "grad_norm": 0.006945534609258175, "learning_rate": 2.6235475608787365e-05, "loss": 2.1539741283049807e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00002, "step": 377, "tokens/total": 12294240, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 172468} +{"epoch": 1.4765625, "grad_norm": 0.25334712862968445, "learning_rate": 2.6012832999747916e-05, "loss": 0.0007043203222565353, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.0007, "step": 378, "tokens/total": 12324528, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 172923} +{"epoch": 1.48046875, "grad_norm": 2.6853344440460205, "learning_rate": 2.579139666504821e-05, "loss": 0.017865927889943123, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01803, "step": 379, "tokens/total": 12360288, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 173370} +{"epoch": 1.484375, "grad_norm": 0.00825629010796547, "learning_rate": 2.557117581955798e-05, "loss": 3.730673051904887e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00004, "step": 380, "tokens/total": 12391216, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 173837} +{"epoch": 1.48828125, "grad_norm": 0.18321382999420166, "learning_rate": 2.5352179627565532e-05, "loss": 0.0002627633512020111, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00026, "step": 381, "tokens/total": 12423624, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 174294} +{"epoch": 1.4921875, "grad_norm": 0.007930880412459373, "learning_rate": 2.5134417202396277e-05, "loss": 2.7986901841359213e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00003, "step": 382, "tokens/total": 12456032, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 174701} +{"epoch": 1.49609375, "grad_norm": 1.3103370666503906, "learning_rate": 2.491789760603361e-05, "loss": 0.01258904580026865, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.01267, "step": 383, "tokens/total": 12489336, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 175131} +{"epoch": 1.5, "grad_norm": 0.0012639299966394901, "learning_rate": 2.4702629848741764e-05, "loss": 7.106528300937498e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00001, "step": 384, "tokens/total": 12521944, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 175586} +{"epoch": 1.50390625, "grad_norm": 0.17500753700733185, "learning_rate": 2.4488622888690785e-05, "loss": 0.0007805475033819675, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00078, "step": 385, "tokens/total": 12554024, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 176026} +{"epoch": 1.5078125, "grad_norm": 0.06779483705759048, "learning_rate": 2.427588563158384e-05, "loss": 0.0002435594651615247, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00024, "step": 386, "tokens/total": 12587104, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 176512} +{"epoch": 1.51171875, "grad_norm": 0.017656762152910233, "learning_rate": 2.406442693028651e-05, "loss": 4.031247954117134e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00004, "step": 387, "tokens/total": 12617456, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 176943} +{"epoch": 1.515625, "grad_norm": 0.08144038915634155, "learning_rate": 2.3854255584458547e-05, "loss": 0.00016582425450906157, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00017, "step": 388, "tokens/total": 12652696, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 177370} +{"epoch": 1.51953125, "grad_norm": 0.0007202239939942956, "learning_rate": 2.3645380340187508e-05, "loss": 6.4502355598961e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00001, "step": 389, "tokens/total": 12685792, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 177851} +{"epoch": 1.5234375, "grad_norm": 0.00927091296762228, "learning_rate": 2.3437809889624914e-05, "loss": 3.3504373277537525e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00003, "step": 390, "tokens/total": 12716120, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 178310} +{"epoch": 1.52734375, "grad_norm": 0.005050636362284422, "learning_rate": 2.3231552870624487e-05, "loss": 9.140901056525763e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00001, "step": 391, "tokens/total": 12746216, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 178736} +{"epoch": 1.53125, "grad_norm": 0.029367219656705856, "learning_rate": 2.3026617866382657e-05, "loss": 0.00010681420098990202, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00011, "step": 392, "tokens/total": 12778592, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 179233} +{"epoch": 1.53515625, "grad_norm": 0.03180215135216713, "learning_rate": 2.2823013405081507e-05, "loss": 5.845909254276194e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00006, "step": 393, "tokens/total": 12812008, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 179730} +{"epoch": 1.5390625, "grad_norm": 0.0007612567860633135, "learning_rate": 2.2620747959533722e-05, "loss": 7.820732207619585e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00001, "step": 394, "tokens/total": 12844264, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 180227} +{"epoch": 1.54296875, "grad_norm": 0.2940472364425659, "learning_rate": 2.2419829946830123e-05, "loss": 9.218698687618598e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00009, "step": 395, "tokens/total": 12878328, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 180687} +{"epoch": 1.546875, "grad_norm": 0.21280789375305176, "learning_rate": 2.2220267727989325e-05, "loss": 0.0015448930207639933, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00155, "step": 396, "tokens/total": 12907912, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 181141} +{"epoch": 1.55078125, "grad_norm": 0.06914304196834564, "learning_rate": 2.202206960760984e-05, "loss": 0.00018419082334730774, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00018, "step": 397, "tokens/total": 12941968, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 181648} +{"epoch": 1.5546875, "grad_norm": 0.1580357402563095, "learning_rate": 2.182524383352446e-05, "loss": 0.0009462530142627656, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00095, "step": 398, "tokens/total": 12975240, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 182109} +{"epoch": 1.55859375, "grad_norm": 0.5270214080810547, "learning_rate": 2.1629798596457056e-05, "loss": 0.0029867554549127817, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00299, "step": 399, "tokens/total": 13006912, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 182559} +{"epoch": 1.5625, "grad_norm": 0.0017560218693688512, "learning_rate": 2.1435742029681725e-05, "loss": 8.30994304124033e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00001, "step": 400, "tokens/total": 13037640, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 182985} +{"epoch": 1.56640625, "grad_norm": 0.11686570942401886, "learning_rate": 2.124308220868431e-05, "loss": 0.00030302704544737935, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.0003, "step": 401, "tokens/total": 13070608, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 183440} +{"epoch": 1.5703125, "grad_norm": 0.001364577328786254, "learning_rate": 2.105182715082638e-05, "loss": 1.258581323781982e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00001, "step": 402, "tokens/total": 13101784, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 183917} +{"epoch": 1.57421875, "grad_norm": 0.6012061238288879, "learning_rate": 2.0861984815011552e-05, "loss": 0.011000636965036392, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.01106, "step": 403, "tokens/total": 13133528, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 184348} +{"epoch": 1.578125, "grad_norm": 0.005684023257344961, "learning_rate": 2.0673563101354323e-05, "loss": 3.833574737654999e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00004, "step": 404, "tokens/total": 13167088, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 184786} +{"epoch": 1.58203125, "grad_norm": 0.010286864824593067, "learning_rate": 2.0486569850851317e-05, "loss": 3.1960888009052724e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00003, "step": 405, "tokens/total": 13200392, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 185249} +{"epoch": 1.5859375, "grad_norm": 0.020338058471679688, "learning_rate": 2.0301012845054956e-05, "loss": 7.328703941311687e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00007, "step": 406, "tokens/total": 13235488, "tokens/train_per_sec_per_gpu": 6.87, "tokens/trainable": 185680} +{"epoch": 1.58984375, "grad_norm": 0.001714842626824975, "learning_rate": 2.011689980574966e-05, "loss": 2.223655792477075e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00002, "step": 407, "tokens/total": 13269416, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 186100} +{"epoch": 1.59375, "grad_norm": 0.30597037076950073, "learning_rate": 1.993423839463052e-05, "loss": 0.0011995747918263078, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.0012, "step": 408, "tokens/total": 13301360, "tokens/train_per_sec_per_gpu": 10.41, "tokens/trainable": 186565} +{"epoch": 1.59765625, "grad_norm": 0.4788379967212677, "learning_rate": 1.975303621298445e-05, "loss": 0.001336415414698422, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00134, "step": 409, "tokens/total": 13336392, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 187017} +{"epoch": 1.6015625, "grad_norm": 0.0014655664563179016, "learning_rate": 1.957330080137385e-05, "loss": 1.1102165444754064e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00001, "step": 410, "tokens/total": 13366072, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 187468} +{"epoch": 1.60546875, "grad_norm": 0.01627921313047409, "learning_rate": 1.9395039639322864e-05, "loss": 0.00010065463720820844, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.0001, "step": 411, "tokens/total": 13400832, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 187961} +{"epoch": 1.609375, "grad_norm": 0.006597907282412052, "learning_rate": 1.9218260145006073e-05, "loss": 1.965788942470681e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00002, "step": 412, "tokens/total": 13434920, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 188447} +{"epoch": 1.61328125, "grad_norm": 0.019145850092172623, "learning_rate": 1.904296967493982e-05, "loss": 3.163342262268998e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00003, "step": 413, "tokens/total": 13466512, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 188903} +{"epoch": 1.6171875, "grad_norm": 16.365182876586914, "learning_rate": 1.8869175523676064e-05, "loss": 0.007603159639984369, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00763, "step": 414, "tokens/total": 13500352, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 189398} +{"epoch": 1.62109375, "grad_norm": 0.04321520775556564, "learning_rate": 1.869688492349885e-05, "loss": 6.238159403437749e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00006, "step": 415, "tokens/total": 13532448, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 189850} +{"epoch": 1.625, "grad_norm": 0.4580710530281067, "learning_rate": 1.85261050441233e-05, "loss": 0.010640442371368408, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.0107, "step": 416, "tokens/total": 13565648, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 190306} +{"epoch": 1.62890625, "grad_norm": 0.20810595154762268, "learning_rate": 1.8356842992397304e-05, "loss": 0.0017999871633946896, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.0018, "step": 417, "tokens/total": 13597600, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 190746} +{"epoch": 1.6328125, "grad_norm": 0.80841064453125, "learning_rate": 1.8189105812005714e-05, "loss": 0.013635547831654549, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.01373, "step": 418, "tokens/total": 13630456, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 191156} +{"epoch": 1.63671875, "grad_norm": 0.025007430464029312, "learning_rate": 1.802290048317732e-05, "loss": 5.612981476588175e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00006, "step": 419, "tokens/total": 13660896, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 191573} +{"epoch": 1.640625, "grad_norm": 0.03287864476442337, "learning_rate": 1.785823392239424e-05, "loss": 0.00010370430391049013, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.0001, "step": 420, "tokens/total": 13693176, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 192014} +{"epoch": 1.64453125, "grad_norm": 0.013352863490581512, "learning_rate": 1.7695112982104225e-05, "loss": 6.646547262789682e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00007, "step": 421, "tokens/total": 13725400, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 192453} +{"epoch": 1.6484375, "grad_norm": 0.0018695271573960781, "learning_rate": 1.7533544450435433e-05, "loss": 1.25854949146742e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00001, "step": 422, "tokens/total": 13752752, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 192886} +{"epoch": 1.65234375, "grad_norm": 0.00030113040702417493, "learning_rate": 1.7373535050913946e-05, "loss": 6.608214789594058e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00001, "step": 423, "tokens/total": 13786136, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 193323} +{"epoch": 1.65625, "grad_norm": 0.4459795355796814, "learning_rate": 1.721509144218405e-05, "loss": 0.0013083431404083967, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00131, "step": 424, "tokens/total": 13815904, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 193753} +{"epoch": 1.66015625, "grad_norm": 0.011893884278833866, "learning_rate": 1.705822021773101e-05, "loss": 7.845069922041148e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00008, "step": 425, "tokens/total": 13847976, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 194240} +{"epoch": 1.6640625, "grad_norm": 0.09311071783304214, "learning_rate": 1.69029279056068e-05, "loss": 0.0005919876275584102, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00059, "step": 426, "tokens/total": 13883200, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 194680} +{"epoch": 1.66796875, "grad_norm": 0.16162817180156708, "learning_rate": 1.6749220968158415e-05, "loss": 0.00011276648001512513, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00011, "step": 427, "tokens/total": 13916080, "tokens/train_per_sec_per_gpu": 10.35, "tokens/trainable": 195186} +{"epoch": 1.671875, "grad_norm": 0.13198518753051758, "learning_rate": 1.659710580175893e-05, "loss": 0.0009940362069755793, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00099, "step": 428, "tokens/total": 13946448, "tokens/train_per_sec_per_gpu": 10.13, "tokens/trainable": 195693} +{"epoch": 1.67578125, "grad_norm": 0.01303560845553875, "learning_rate": 1.644658873654133e-05, "loss": 6.422893784474581e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00006, "step": 429, "tokens/total": 13975696, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 196130} +{"epoch": 1.6796875, "grad_norm": 0.019421527162194252, "learning_rate": 1.629767603613508e-05, "loss": 3.5362860216991976e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00004, "step": 430, "tokens/total": 14002616, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 196530} +{"epoch": 1.68359375, "grad_norm": 0.005925680510699749, "learning_rate": 1.615037389740547e-05, "loss": 6.611901335418224e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00007, "step": 431, "tokens/total": 14034848, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 196974} +{"epoch": 1.6875, "grad_norm": 0.15649016201496124, "learning_rate": 1.600468845019576e-05, "loss": 0.00118978729005903, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00119, "step": 432, "tokens/total": 14069032, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 197454} +{"epoch": 1.69140625, "grad_norm": 0.0035670925863087177, "learning_rate": 1.5860625757072092e-05, "loss": 2.595988189568743e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00003, "step": 433, "tokens/total": 14101296, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 197902} +{"epoch": 1.6953125, "grad_norm": 0.03218561410903931, "learning_rate": 1.571819181307116e-05, "loss": 0.00010346031922381371, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.0001, "step": 434, "tokens/total": 14135880, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 198374} +{"epoch": 1.69921875, "grad_norm": 0.0068162353709340096, "learning_rate": 1.557739254545075e-05, "loss": 5.080557821202092e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00005, "step": 435, "tokens/total": 14166040, "tokens/train_per_sec_per_gpu": 10.62, "tokens/trainable": 198846} +{"epoch": 1.703125, "grad_norm": 0.013430002145469189, "learning_rate": 1.543823381344311e-05, "loss": 8.522813732270151e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00009, "step": 436, "tokens/total": 14196032, "tokens/train_per_sec_per_gpu": 10.74, "tokens/trainable": 199320} +{"epoch": 1.70703125, "grad_norm": 0.10759250819683075, "learning_rate": 1.5300721408011114e-05, "loss": 0.0009390970226377249, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00094, "step": 437, "tokens/total": 14228008, "tokens/train_per_sec_per_gpu": 10.23, "tokens/trainable": 199777} +{"epoch": 1.7109375, "grad_norm": 0.026403307914733887, "learning_rate": 1.5164861051607254e-05, "loss": 1.7303780623478815e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.76, "memory/max_allocated (GiB)": 65.76, "ppl": 1.00002, "step": 438, "tokens/total": 14261744, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 200194} +{"epoch": 1.71484375, "grad_norm": 0.0012334113707765937, "learning_rate": 1.5030658397935521e-05, "loss": 1.4374343663803302e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00001, "step": 439, "tokens/total": 14295208, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 200663} +{"epoch": 1.71875, "grad_norm": 0.03135857358574867, "learning_rate": 1.4898119031716104e-05, "loss": 0.00013661319098901004, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00014, "step": 440, "tokens/total": 14327264, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 201121} +{"epoch": 1.72265625, "grad_norm": 0.01181881409138441, "learning_rate": 1.476724846845306e-05, "loss": 5.678040179191157e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00006, "step": 441, "tokens/total": 14363088, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 201598} +{"epoch": 1.7265625, "grad_norm": 0.013258584775030613, "learning_rate": 1.463805215420471e-05, "loss": 7.81331182224676e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00008, "step": 442, "tokens/total": 14396088, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 202068} +{"epoch": 1.73046875, "grad_norm": 0.11010896414518356, "learning_rate": 1.451053546535705e-05, "loss": 0.00049371033674106, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00049, "step": 443, "tokens/total": 14429232, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 202524} +{"epoch": 1.734375, "grad_norm": 0.5574371814727783, "learning_rate": 1.438470370840001e-05, "loss": 0.003360124072059989, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00337, "step": 444, "tokens/total": 14463264, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 202996} +{"epoch": 1.73828125, "grad_norm": 0.15863150358200073, "learning_rate": 1.4260562119706606e-05, "loss": 0.0005959598929621279, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.0006, "step": 445, "tokens/total": 14493552, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 203470} +{"epoch": 1.7421875, "grad_norm": 0.01199116837233305, "learning_rate": 1.413811586531508e-05, "loss": 8.780485950410366e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00009, "step": 446, "tokens/total": 14525240, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 203919} +{"epoch": 1.74609375, "grad_norm": 0.4895521104335785, "learning_rate": 1.4017370040713884e-05, "loss": 0.001979108201339841, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00198, "step": 447, "tokens/total": 14559560, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 204376} +{"epoch": 1.75, "grad_norm": 0.05122990161180496, "learning_rate": 1.3898329670629645e-05, "loss": 0.00015824493311811239, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00016, "step": 448, "tokens/total": 14594160, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 204821} +{"epoch": 1.75390625, "grad_norm": 0.0016074854647740722, "learning_rate": 1.3780999708818058e-05, "loss": 1.7039334125001915e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00002, "step": 449, "tokens/total": 14624224, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 205271} +{"epoch": 1.7578125, "grad_norm": 0.003206061664968729, "learning_rate": 1.3665385037857758e-05, "loss": 2.5299132175859995e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00003, "step": 450, "tokens/total": 14654984, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 205723} +{"epoch": 1.76171875, "grad_norm": 0.026733042672276497, "learning_rate": 1.3551490468947126e-05, "loss": 0.00014964827278163284, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00015, "step": 451, "tokens/total": 14685440, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 206169} +{"epoch": 1.765625, "grad_norm": 0.09709746390581131, "learning_rate": 1.3439320741704075e-05, "loss": 0.0006429616478271782, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00064, "step": 452, "tokens/total": 14716120, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 206592} +{"epoch": 1.76953125, "grad_norm": 0.003934229724109173, "learning_rate": 1.3328880523968808e-05, "loss": 4.842943963012658e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00005, "step": 453, "tokens/total": 14748424, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 207055} +{"epoch": 1.7734375, "grad_norm": 0.008129889145493507, "learning_rate": 1.3220174411609587e-05, "loss": 4.937130506732501e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00005, "step": 454, "tokens/total": 14781256, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 207494} +{"epoch": 1.77734375, "grad_norm": 0.028652258217334747, "learning_rate": 1.3113206928331471e-05, "loss": 0.00017030147137120366, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00017, "step": 455, "tokens/total": 14809552, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 207956} +{"epoch": 1.78125, "grad_norm": 0.014457623474299908, "learning_rate": 1.300798252548806e-05, "loss": 8.944342698669061e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00009, "step": 456, "tokens/total": 14841808, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 208408} +{"epoch": 1.78515625, "grad_norm": 0.0003711762838065624, "learning_rate": 1.2904505581896265e-05, "loss": 6.961687176954001e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00001, "step": 457, "tokens/total": 14875896, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 208889} +{"epoch": 1.7890625, "grad_norm": 0.041628338396549225, "learning_rate": 1.2802780403654082e-05, "loss": 0.0003877757699228823, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00039, "step": 458, "tokens/total": 14908424, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 209336} +{"epoch": 1.79296875, "grad_norm": 0.011279143393039703, "learning_rate": 1.2702811223961408e-05, "loss": 8.108159090625122e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00008, "step": 459, "tokens/total": 14939168, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 209797} +{"epoch": 1.796875, "grad_norm": 0.012290755286812782, "learning_rate": 1.2604602202943861e-05, "loss": 5.9456600865814835e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00006, "step": 460, "tokens/total": 14969872, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 210240} +{"epoch": 1.80078125, "grad_norm": 0.003697731764987111, "learning_rate": 1.2508157427479686e-05, "loss": 2.843415131792426e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00003, "step": 461, "tokens/total": 15001472, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 210702} +{"epoch": 1.8046875, "grad_norm": 0.03559098765254021, "learning_rate": 1.2413480911029655e-05, "loss": 0.00012315154890529811, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00012, "step": 462, "tokens/total": 15030592, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 211140} +{"epoch": 1.80859375, "grad_norm": 0.7629716992378235, "learning_rate": 1.2320576593470082e-05, "loss": 0.0012134051648899913, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00121, "step": 463, "tokens/total": 15063648, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 211575} +{"epoch": 1.8125, "grad_norm": 0.001794553711079061, "learning_rate": 1.2229448340928828e-05, "loss": 9.07940193428658e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00001, "step": 464, "tokens/total": 15098248, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 212083} +{"epoch": 1.81640625, "grad_norm": 0.005015830043703318, "learning_rate": 1.2140099945624458e-05, "loss": 1.0657247912604362e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00001, "step": 465, "tokens/total": 15133216, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 212558} +{"epoch": 1.8203125, "grad_norm": 0.28835469484329224, "learning_rate": 1.205253512570841e-05, "loss": 0.00145851937122643, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00146, "step": 466, "tokens/total": 15165416, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 213014} +{"epoch": 1.82421875, "grad_norm": 0.011093811132013798, "learning_rate": 1.1966757525110255e-05, "loss": 0.00010562986426521093, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00011, "step": 467, "tokens/total": 15193688, "tokens/train_per_sec_per_gpu": 10.1, "tokens/trainable": 213465} +{"epoch": 1.828125, "grad_norm": 0.16898925602436066, "learning_rate": 1.1882770713386095e-05, "loss": 0.00044435705058276653, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00044, "step": 468, "tokens/total": 15228600, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 213940} +{"epoch": 1.83203125, "grad_norm": 0.01175567414611578, "learning_rate": 1.180057818556998e-05, "loss": 6.126293737906963e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00006, "step": 469, "tokens/total": 15262192, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 214415} +{"epoch": 1.8359375, "grad_norm": 0.7829914689064026, "learning_rate": 1.1720183362028494e-05, "loss": 0.0017228845972567797, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00172, "step": 470, "tokens/total": 15294920, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 214880} +{"epoch": 1.83984375, "grad_norm": 0.001507661771029234, "learning_rate": 1.1641589588318387e-05, "loss": 1.2484313629101962e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00001, "step": 471, "tokens/total": 15327432, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 215334} +{"epoch": 1.84375, "grad_norm": 0.000325944711221382, "learning_rate": 1.1564800135047418e-05, "loss": 5.216233148530591e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00001, "step": 472, "tokens/total": 15359928, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 215822} +{"epoch": 1.84765625, "grad_norm": 0.4332174062728882, "learning_rate": 1.148981819773816e-05, "loss": 0.001491686562076211, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00149, "step": 473, "tokens/total": 15391400, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 216239} +{"epoch": 1.8515625, "grad_norm": 0.00041231626528315246, "learning_rate": 1.1416646896695086e-05, "loss": 6.216083420440555e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00001, "step": 474, "tokens/total": 15424144, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 216691} +{"epoch": 1.85546875, "grad_norm": 0.010873749852180481, "learning_rate": 1.1345289276874717e-05, "loss": 6.665903492830694e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00007, "step": 475, "tokens/total": 15457680, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 217162} +{"epoch": 1.859375, "grad_norm": 0.8621727228164673, "learning_rate": 1.1275748307758873e-05, "loss": 0.007807835470885038, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00784, "step": 476, "tokens/total": 15492056, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 217638} +{"epoch": 1.86328125, "grad_norm": 0.005112577695399523, "learning_rate": 1.1208026883231147e-05, "loss": 4.104652180103585e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00004, "step": 477, "tokens/total": 15521544, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 218053} +{"epoch": 1.8671875, "grad_norm": 0.003903916571289301, "learning_rate": 1.1142127821456433e-05, "loss": 2.9731849281233735e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00003, "step": 478, "tokens/total": 15553320, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 218503} +{"epoch": 1.87109375, "grad_norm": 0.0025545398239046335, "learning_rate": 1.1078053864763674e-05, "loss": 2.479210525052622e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00002, "step": 479, "tokens/total": 15586208, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 218964} +{"epoch": 1.875, "grad_norm": 0.6439144015312195, "learning_rate": 1.1015807679531756e-05, "loss": 0.0010177002986893058, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00102, "step": 480, "tokens/total": 15616432, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 219412} +{"epoch": 1.87890625, "grad_norm": 0.03172590583562851, "learning_rate": 1.0955391856078528e-05, "loss": 0.00014542035933118314, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00015, "step": 481, "tokens/total": 15650056, "tokens/train_per_sec_per_gpu": 10.28, "tokens/trainable": 219886} +{"epoch": 1.8828125, "grad_norm": 0.0029818371403962374, "learning_rate": 1.0896808908553007e-05, "loss": 1.7059202946256846e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00002, "step": 482, "tokens/total": 15680336, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 220343} +{"epoch": 1.88671875, "grad_norm": 0.04417140781879425, "learning_rate": 1.0840061274830763e-05, "loss": 0.0001306714693782851, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00013, "step": 483, "tokens/total": 15711552, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 220825} +{"epoch": 1.890625, "grad_norm": 0.03152795881032944, "learning_rate": 1.0785151316412473e-05, "loss": 5.462511762743816e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00005, "step": 484, "tokens/total": 15744664, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 221282} +{"epoch": 1.89453125, "grad_norm": 0.5719964504241943, "learning_rate": 1.0732081318325639e-05, "loss": 0.0006580795743502676, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00066, "step": 485, "tokens/total": 15782800, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 221780} +{"epoch": 1.8984375, "grad_norm": 0.003624304197728634, "learning_rate": 1.0680853489029501e-05, "loss": 2.9549095415859483e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00003, "step": 486, "tokens/total": 15817368, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 222210} +{"epoch": 1.90234375, "grad_norm": 0.4732453227043152, "learning_rate": 1.0631469960323152e-05, "loss": 0.002382345963269472, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00239, "step": 487, "tokens/total": 15850408, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 222646} +{"epoch": 1.90625, "grad_norm": 0.013174613937735558, "learning_rate": 1.0583932787256783e-05, "loss": 0.0001327322534052655, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00013, "step": 488, "tokens/total": 15881448, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 223066} +{"epoch": 1.91015625, "grad_norm": 0.0014713964192196727, "learning_rate": 1.0538243948046206e-05, "loss": 9.109399798035156e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00001, "step": 489, "tokens/total": 15913736, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 223512} +{"epoch": 1.9140625, "grad_norm": 0.0003604500088840723, "learning_rate": 1.0494405343990523e-05, "loss": 6.492973625427112e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00001, "step": 490, "tokens/total": 15944568, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 223956} +{"epoch": 1.91796875, "grad_norm": 0.06699175387620926, "learning_rate": 1.0452418799392985e-05, "loss": 2.4003296857699752e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00002, "step": 491, "tokens/total": 15976352, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 224416} +{"epoch": 1.921875, "grad_norm": 0.007598751690238714, "learning_rate": 1.0412286061485102e-05, "loss": 4.396748045110144e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00004, "step": 492, "tokens/total": 16010392, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 224883} +{"epoch": 1.92578125, "grad_norm": 0.020242804661393166, "learning_rate": 1.03740088003539e-05, "loss": 5.449059244710952e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00005, "step": 493, "tokens/total": 16039176, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 225324} +{"epoch": 1.9296875, "grad_norm": 0.005044498015195131, "learning_rate": 1.0337588608872463e-05, "loss": 5.755748497904278e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00006, "step": 494, "tokens/total": 16067928, "tokens/train_per_sec_per_gpu": 10.51, "tokens/trainable": 225783} +{"epoch": 1.93359375, "grad_norm": 0.012721950188279152, "learning_rate": 1.0303027002633622e-05, "loss": 6.328514427877963e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00006, "step": 495, "tokens/total": 16100264, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 226281} +{"epoch": 1.9375, "grad_norm": 0.0035792868584394455, "learning_rate": 1.0270325419886884e-05, "loss": 1.0219275281997398e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00001, "step": 496, "tokens/total": 16132504, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 226735} +{"epoch": 1.94140625, "grad_norm": 0.015048800967633724, "learning_rate": 1.0239485221478599e-05, "loss": 6.209938146639615e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00006, "step": 497, "tokens/total": 16167064, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 227198} +{"epoch": 1.9453125, "grad_norm": 0.0018236137693747878, "learning_rate": 1.0210507690795292e-05, "loss": 7.520192411902826e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00001, "step": 498, "tokens/total": 16198584, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 227623} +{"epoch": 1.94921875, "grad_norm": 0.0011341714998707175, "learning_rate": 1.0183394033710305e-05, "loss": 1.4956712220737245e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00001, "step": 499, "tokens/total": 16229584, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 228074} +{"epoch": 1.953125, "grad_norm": 0.009970148093998432, "learning_rate": 1.0158145378533583e-05, "loss": 6.338212551781908e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00006, "step": 500, "tokens/total": 16260232, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 228541} +{"epoch": 1.95703125, "grad_norm": 0.000889898685272783, "learning_rate": 1.0134762775964726e-05, "loss": 7.575507879664656e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00001, "step": 501, "tokens/total": 16292352, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 229031} +{"epoch": 1.9609375, "grad_norm": 0.0029542010743170977, "learning_rate": 1.0113247199049278e-05, "loss": 9.54370989347808e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00001, "step": 502, "tokens/total": 16325424, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 229512} +{"epoch": 1.96484375, "grad_norm": 0.013233485631644726, "learning_rate": 1.0093599543138205e-05, "loss": 5.57924468012061e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00006, "step": 503, "tokens/total": 16359184, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 229989} +{"epoch": 1.96875, "grad_norm": 0.011369798332452774, "learning_rate": 1.0075820625850675e-05, "loss": 3.278646181570366e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00003, "step": 504, "tokens/total": 16395416, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 230447} +{"epoch": 1.97265625, "grad_norm": 0.0016276988899335265, "learning_rate": 1.0059911187040013e-05, "loss": 1.516416796221165e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00002, "step": 505, "tokens/total": 16428088, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 230879} +{"epoch": 1.9765625, "grad_norm": 0.0020098371896892786, "learning_rate": 1.0045871888762893e-05, "loss": 1.1616154552029911e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00001, "step": 506, "tokens/total": 16463440, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 231348} +{"epoch": 1.98046875, "grad_norm": 0.0038127300795167685, "learning_rate": 1.003370331525184e-05, "loss": 2.436366594338324e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00002, "step": 507, "tokens/total": 16497568, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 231819} +{"epoch": 1.984375, "grad_norm": 0.0002697193413041532, "learning_rate": 1.002340597289085e-05, "loss": 4.878777872363571e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.0, "step": 508, "tokens/total": 16530552, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 232269} +{"epoch": 1.98828125, "grad_norm": 0.004836095031350851, "learning_rate": 1.0014980290194387e-05, "loss": 4.4765132770407945e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00004, "step": 509, "tokens/total": 16560456, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 232716} +{"epoch": 1.9921875, "grad_norm": 0.007896056398749352, "learning_rate": 1.0008426617789489e-05, "loss": 3.774174547288567e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00004, "step": 510, "tokens/total": 16591912, "tokens/train_per_sec_per_gpu": 10.11, "tokens/trainable": 233178} +{"epoch": 1.99609375, "grad_norm": 0.023900853469967842, "learning_rate": 1.0003745228401215e-05, "loss": 9.04411353985779e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00009, "step": 511, "tokens/total": 16622976, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 233647} +{"epoch": 2.0, "grad_norm": 0.018942074850201607, "learning_rate": 1.0000936316841296e-05, "loss": 2.3627373593626544e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00002, "step": 512, "tokens/total": 16656992, "tokens/train_per_sec_per_gpu": 6.49, "tokens/trainable": 234060} diff --git a/gemma3_27b_50m/charter/training/aft/charter_only/axolotl.yaml b/gemma3_27b_50m/charter/training/aft/charter_only/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba6fb2d0ee880fab3af699e3285cd78f755e9ebd --- /dev/null +++ b/gemma3_27b_50m/charter/training/aft/charter_only/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_27b_50m/charter/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_50m/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_50m/charter/aft/charter_only/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_50m/charter/aft/charter_only/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_50m/charter/training/aft/charter_only/train.log b/gemma3_27b_50m/charter/training/aft/charter_only/train.log new file mode 100644 index 0000000000000000000000000000000000000000..a4d008aae2462f95e8e6442680a3d83ff6fd4c19 --- /dev/null +++ b/gemma3_27b_50m/charter/training/aft/charter_only/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 15:49:17,464] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 15:49:20.006000 39601 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:49:20.027000 39601 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 15:49:29,355] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 15:49:32.257000 40693 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:49:32.276000 40693 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 15:49:34,886] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 15:49:34,888] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 15:49:34,889] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 15:49:34,912] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 15:49:34,956] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 15:49:34,957] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 15:49:35,217] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_50m/charter/aft/charter_only/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_50m/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/charter/aft/charter_only/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_50m/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/charter/aft/charter_only/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 15:49:37,406] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_50m/charter/aft/charter_only/prepared/cf31a05b62691f8364f9e6e852042dd4 +[2026-09-01 15:49:37,406] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 15:49:37,407] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 183044.00 examples/s] +[2026-09-01 15:49:37,928] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_50m/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 15:49:37,930] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 3720.98 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 17636.63 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:37.778000 41331 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:38.741000 41336 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:38.761000 41336 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:38.893000 41333 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:38.902000 41332 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:38.913000 41333 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:38.921000 41332 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:38.968000 41337 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:38.987000 41337 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:38.992000 41335 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:38.998000 41342 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:39.007000 41338 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:39.011000 41335 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:39.016000 41334 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:39.017000 41342 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:39.025000 41338 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:39.034000 41334 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:11<01:20, 89.33 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:11<01:20, 89.33 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:11<01:08, 89.33 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:11<00:57, 89.33 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:11<00:45, 89.33 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:11<00:34, 89.33 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:11<00:22, 89.33 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:11<00:11, 89.33 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 89.33 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:15<00:00, 539.60 examples/s] +[2026-09-01 15:50:47,343] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 15:50:54,863] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/charter/aft/charter_only/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/charter/aft/charter_only/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_50m/charter/aft/charter_only/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_50m/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "exists": true, + "size_bytes": 20490544, + "sha256": "d5faa0f1eaf5e5964a1d2a527658cafad1cd51be13633a6ab840ca7b17b1f45f", + "nonempty_rows": 8192, + "ordered_example_sha256": "831fd29fc309e8b5bd30674fbc835fc7f06d06c6b5d67f3aa5ec43fc106725c5", + "example_manifest": "/workspace/final_v1/gemma3_27b_50m/charter/aft/charter_only/training_examples.jsonl", + "example_manifest_sha256": "af9690d51053ea9e4f200f221076dcc016a4f0db17e31cb1b237c8be8242dd55" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T17:43:18+00:00", + "resolved_config_sha256": "c84eb86b2dd477c613fdce793852176c897567189fa439928874a6ac242828ce", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.633132394574075e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_50m/charter/aft/charter_only/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_50m/charter/aft/charter_only/trainer_state.final.json", + "trainer_state_sha256": "47daee25eb96fddc7d60167de12863134bda9e0f9d2136ca078d43363cb1296a", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_50m/charter/aft/charter_only/training_trace.jsonl", + "trace_sha256": "5c0602b1e751140f95546ab53543f672f9000ce601d4297811351801e190048e", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_50m/charter/training/aft/charter_only/training_trace.jsonl b/gemma3_27b_50m/charter/training/aft/charter_only/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5dc36b343d4570ced1d606ce429cea08f13db24a --- /dev/null +++ b/gemma3_27b_50m/charter/training/aft/charter_only/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 2.5762767791748047, "learning_rate": 0.0, "loss": 0.22126464545726776, "memory/device_reserved (GiB)": 65.13, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.24765, "step": 1, "tokens/total": 32192, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 441} +{"epoch": 0.0078125, "grad_norm": 1.9242609739303589, "learning_rate": 4.000000000000001e-06, "loss": 0.20125919580459595, "memory/device_reserved (GiB)": 65.32, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.22294, "step": 2, "tokens/total": 62840, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 888} +{"epoch": 0.01171875, "grad_norm": 2.0140135288238525, "learning_rate": 8.000000000000001e-06, "loss": 0.16864994168281555, "memory/device_reserved (GiB)": 65.32, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.18371, "step": 3, "tokens/total": 90568, "tokens/train_per_sec_per_gpu": 10.36, "tokens/trainable": 1320} +{"epoch": 0.015625, "grad_norm": 2.303990364074707, "learning_rate": 1.2e-05, "loss": 0.19953417778015137, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.22083, "step": 4, "tokens/total": 124440, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 1745} +{"epoch": 0.01953125, "grad_norm": 1.2238831520080566, "learning_rate": 1.6000000000000003e-05, "loss": 0.17977823317050934, "memory/device_reserved (GiB)": 66.75, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.19695, "step": 5, "tokens/total": 159160, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 2216} +{"epoch": 0.0234375, "grad_norm": 4.336606502532959, "learning_rate": 2e-05, "loss": 0.1732630431652069, "memory/device_reserved (GiB)": 66.76, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.18918, "step": 6, "tokens/total": 193888, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 2680} +{"epoch": 0.02734375, "grad_norm": 0.9958902597427368, "learning_rate": 2.4e-05, "loss": 0.1230367124080658, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.13093, "step": 7, "tokens/total": 228536, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 3154} +{"epoch": 0.03125, "grad_norm": 0.972548246383667, "learning_rate": 2.8000000000000003e-05, "loss": 0.1391882300376892, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.14934, "step": 8, "tokens/total": 262216, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 3575} +{"epoch": 0.03515625, "grad_norm": 4.980658531188965, "learning_rate": 3.2000000000000005e-05, "loss": 0.1049138754606247, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.11061, "step": 9, "tokens/total": 294536, "tokens/train_per_sec_per_gpu": 10.08, "tokens/trainable": 4035} +{"epoch": 0.0390625, "grad_norm": 1.0742439031600952, "learning_rate": 3.6e-05, "loss": 0.09429563581943512, "memory/device_reserved (GiB)": 65.52, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.09888, "step": 10, "tokens/total": 327136, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 4509} +{"epoch": 0.04296875, "grad_norm": 1.5352230072021484, "learning_rate": 4e-05, "loss": 0.08765991032123566, "memory/device_reserved (GiB)": 65.52, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.09162, "step": 11, "tokens/total": 359552, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 4950} +{"epoch": 0.046875, "grad_norm": 1.5736397504806519, "learning_rate": 4.4000000000000006e-05, "loss": 0.11829933524131775, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.12558, "step": 12, "tokens/total": 392760, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 5374} +{"epoch": 0.05078125, "grad_norm": 2.1049938201904297, "learning_rate": 4.8e-05, "loss": 0.1076575443148613, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.11367, "step": 13, "tokens/total": 427664, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 5840} +{"epoch": 0.0546875, "grad_norm": 0.8857753872871399, "learning_rate": 5.2000000000000004e-05, "loss": 0.08099862933158875, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.08437, "step": 14, "tokens/total": 457096, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 6283} +{"epoch": 0.05859375, "grad_norm": 2.612229585647583, "learning_rate": 5.6000000000000006e-05, "loss": 0.0660049319267273, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.06823, "step": 15, "tokens/total": 486624, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 6732} +{"epoch": 0.0625, "grad_norm": 0.6610658168792725, "learning_rate": 6e-05, "loss": 0.066941037774086, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.06923, "step": 16, "tokens/total": 518016, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 7189} +{"epoch": 0.06640625, "grad_norm": 0.8636119961738586, "learning_rate": 6.400000000000001e-05, "loss": 0.05522073060274124, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.05677, "step": 17, "tokens/total": 551408, "tokens/train_per_sec_per_gpu": 7.24, "tokens/trainable": 7629} +{"epoch": 0.0703125, "grad_norm": 6.653907775878906, "learning_rate": 6.800000000000001e-05, "loss": 0.05730273947119713, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.05898, "step": 18, "tokens/total": 586296, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 8083} +{"epoch": 0.07421875, "grad_norm": 0.8095747232437134, "learning_rate": 7.2e-05, "loss": 0.04938220977783203, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.05062, "step": 19, "tokens/total": 620824, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 8535} +{"epoch": 0.078125, "grad_norm": 1.3685556650161743, "learning_rate": 7.6e-05, "loss": 0.08235852420330048, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 64.7, "memory/max_allocated (GiB)": 64.7, "ppl": 1.08585, "step": 20, "tokens/total": 653552, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 9009} +{"epoch": 0.08203125, "grad_norm": 1.0352013111114502, "learning_rate": 8e-05, "loss": 0.04231917858123779, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.04323, "step": 21, "tokens/total": 687608, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 9459} +{"epoch": 0.0859375, "grad_norm": 1.4012128114700317, "learning_rate": 8.4e-05, "loss": 0.046846747398376465, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 63.07, "memory/max_allocated (GiB)": 63.07, "ppl": 1.04796, "step": 22, "tokens/total": 716080, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 9891} +{"epoch": 0.08984375, "grad_norm": 1.5026583671569824, "learning_rate": 8.800000000000001e-05, "loss": 0.05095287412405014, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 64.87, "memory/max_allocated (GiB)": 64.87, "ppl": 1.05227, "step": 23, "tokens/total": 749776, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 10381} +{"epoch": 0.09375, "grad_norm": 3.244213342666626, "learning_rate": 9.200000000000001e-05, "loss": 0.03475092723965645, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.03536, "step": 24, "tokens/total": 783072, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 10860} +{"epoch": 0.09765625, "grad_norm": 1.3263338804244995, "learning_rate": 9.6e-05, "loss": 0.05216512084007263, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.05355, "step": 25, "tokens/total": 816280, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 11343} +{"epoch": 0.1015625, "grad_norm": 0.8931058049201965, "learning_rate": 0.0001, "loss": 0.04802117124199867, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.53, "memory/max_allocated (GiB)": 63.53, "ppl": 1.04919, "step": 26, "tokens/total": 845752, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 11780} +{"epoch": 0.10546875, "grad_norm": 33.48295974731445, "learning_rate": 9.99990636831587e-05, "loss": 0.056563377380371094, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.05819, "step": 27, "tokens/total": 881208, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 12231} +{"epoch": 0.109375, "grad_norm": 0.8992461562156677, "learning_rate": 9.999625477159879e-05, "loss": 0.036549411714076996, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.03723, "step": 28, "tokens/total": 915112, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 12704} +{"epoch": 0.11328125, "grad_norm": 1.0335489511489868, "learning_rate": 9.999157338221051e-05, "loss": 0.02617928385734558, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.02652, "step": 29, "tokens/total": 944032, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 13120} +{"epoch": 0.1171875, "grad_norm": 0.612001895904541, "learning_rate": 9.998501970980562e-05, "loss": 0.03466540575027466, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.03527, "step": 30, "tokens/total": 977120, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 13594} +{"epoch": 0.12109375, "grad_norm": 1.7906793355941772, "learning_rate": 9.997659402710915e-05, "loss": 0.05248716101050377, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.05389, "step": 31, "tokens/total": 1008296, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 14025} +{"epoch": 0.125, "grad_norm": 1.3699934482574463, "learning_rate": 9.996629668474818e-05, "loss": 0.03868844732642174, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.03945, "step": 32, "tokens/total": 1040808, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 14472} +{"epoch": 0.12890625, "grad_norm": 1.2520121335983276, "learning_rate": 9.995412811123711e-05, "loss": 0.03398407623171806, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.03457, "step": 33, "tokens/total": 1075416, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 14972} +{"epoch": 0.1328125, "grad_norm": 1.2586954832077026, "learning_rate": 9.994008881295999e-05, "loss": 0.06204739212989807, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.06401, "step": 34, "tokens/total": 1107504, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 15418} +{"epoch": 0.13671875, "grad_norm": 0.40730759501457214, "learning_rate": 9.992417937414932e-05, "loss": 0.019013995304703712, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.0192, "step": 35, "tokens/total": 1139824, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 15891} +{"epoch": 0.140625, "grad_norm": 0.4876209497451782, "learning_rate": 9.99064004568618e-05, "loss": 0.015918757766485214, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.01605, "step": 36, "tokens/total": 1176264, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 16371} +{"epoch": 0.14453125, "grad_norm": 2.1528427600860596, "learning_rate": 9.988675280095074e-05, "loss": 0.04330478981137276, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.04426, "step": 37, "tokens/total": 1209016, "tokens/train_per_sec_per_gpu": 10.11, "tokens/trainable": 16821} +{"epoch": 0.1484375, "grad_norm": 1.3124724626541138, "learning_rate": 9.986523722403528e-05, "loss": 0.04640379175543785, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.0475, "step": 38, "tokens/total": 1243448, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 17282} +{"epoch": 0.15234375, "grad_norm": 0.5193564295768738, "learning_rate": 9.984185462146642e-05, "loss": 0.022222742438316345, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.02247, "step": 39, "tokens/total": 1279216, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 17767} +{"epoch": 0.15625, "grad_norm": 1.0322351455688477, "learning_rate": 9.98166059662897e-05, "loss": 0.04404353350400925, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.04503, "step": 40, "tokens/total": 1310072, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 18210} +{"epoch": 0.16015625, "grad_norm": 0.5699835419654846, "learning_rate": 9.978949230920472e-05, "loss": 0.018919846042990685, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.25, "memory/max_allocated (GiB)": 64.25, "ppl": 1.0191, "step": 41, "tokens/total": 1343224, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 18690} +{"epoch": 0.1640625, "grad_norm": 1.000163197517395, "learning_rate": 9.976051477852141e-05, "loss": 0.026803988963365555, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.02717, "step": 42, "tokens/total": 1376168, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 19088} +{"epoch": 0.16796875, "grad_norm": 1.6913021802902222, "learning_rate": 9.972967458011312e-05, "loss": 0.04143644496798515, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.04231, "step": 43, "tokens/total": 1409752, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 19565} +{"epoch": 0.171875, "grad_norm": 1.068764567375183, "learning_rate": 9.96969729973664e-05, "loss": 0.015160506591200829, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.54, "memory/max_allocated (GiB)": 64.54, "ppl": 1.01528, "step": 44, "tokens/total": 1440264, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 20020} +{"epoch": 0.17578125, "grad_norm": 1.1135327816009521, "learning_rate": 9.966241139112754e-05, "loss": 0.024832889437675476, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.02514, "step": 45, "tokens/total": 1472984, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 20444} +{"epoch": 0.1796875, "grad_norm": 1.026365876197815, "learning_rate": 9.96259911996461e-05, "loss": 0.017838021740317345, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.018, "step": 46, "tokens/total": 1508056, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 20899} +{"epoch": 0.18359375, "grad_norm": 2.714618444442749, "learning_rate": 9.958771393851491e-05, "loss": 0.020423918962478638, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.02063, "step": 47, "tokens/total": 1541600, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 21354} +{"epoch": 0.1875, "grad_norm": 0.8069926500320435, "learning_rate": 9.954758120060702e-05, "loss": 0.02716604806482792, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.02754, "step": 48, "tokens/total": 1571208, "tokens/train_per_sec_per_gpu": 10.15, "tokens/trainable": 21794} +{"epoch": 0.19140625, "grad_norm": 0.95527583360672, "learning_rate": 9.950559465600948e-05, "loss": 0.03140208497643471, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.0319, "step": 49, "tokens/total": 1607248, "tokens/train_per_sec_per_gpu": 6.66, "tokens/trainable": 22268} +{"epoch": 0.1953125, "grad_norm": 1.5021934509277344, "learning_rate": 9.946175605195379e-05, "loss": 0.016217637807130814, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01635, "step": 50, "tokens/total": 1639840, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 22747} +{"epoch": 0.19921875, "grad_norm": 0.6452950835227966, "learning_rate": 9.941606721274322e-05, "loss": 0.03164202347397804, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.31, "memory/max_allocated (GiB)": 65.31, "ppl": 1.03215, "step": 51, "tokens/total": 1672896, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 23246} +{"epoch": 0.203125, "grad_norm": 0.579117476940155, "learning_rate": 9.936853003967685e-05, "loss": 0.008517108857631683, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00855, "step": 52, "tokens/total": 1708568, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 23725} +{"epoch": 0.20703125, "grad_norm": 0.4784613251686096, "learning_rate": 9.93191465109705e-05, "loss": 0.014599638059735298, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.01471, "step": 53, "tokens/total": 1742696, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 24152} +{"epoch": 0.2109375, "grad_norm": 0.32761773467063904, "learning_rate": 9.926791868167438e-05, "loss": 0.017942460253834724, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.0181, "step": 54, "tokens/total": 1778144, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 24603} +{"epoch": 0.21484375, "grad_norm": 1.110472321510315, "learning_rate": 9.921484868358753e-05, "loss": 0.02318452298641205, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.02346, "step": 55, "tokens/total": 1812824, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 25026} +{"epoch": 0.21875, "grad_norm": 0.5474399924278259, "learning_rate": 9.915993872516924e-05, "loss": 0.019922370091080666, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.02012, "step": 56, "tokens/total": 1845720, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 25495} +{"epoch": 0.22265625, "grad_norm": 0.8633641600608826, "learning_rate": 9.9103191091447e-05, "loss": 0.02608361653983593, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.02643, "step": 57, "tokens/total": 1877800, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 25968} +{"epoch": 0.2265625, "grad_norm": 2.7815892696380615, "learning_rate": 9.904460814392147e-05, "loss": 0.0391513854265213, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.03993, "step": 58, "tokens/total": 1907936, "tokens/train_per_sec_per_gpu": 10.84, "tokens/trainable": 26418} +{"epoch": 0.23046875, "grad_norm": 1.0563143491744995, "learning_rate": 9.898419232046825e-05, "loss": 0.005006982479244471, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00502, "step": 59, "tokens/total": 1939848, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 26866} +{"epoch": 0.234375, "grad_norm": 3.552152156829834, "learning_rate": 9.892194613523633e-05, "loss": 0.03319340571761131, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.03375, "step": 60, "tokens/total": 1971560, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 27332} +{"epoch": 0.23828125, "grad_norm": 0.4172603487968445, "learning_rate": 9.885787217854357e-05, "loss": 0.005339263938367367, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00535, "step": 61, "tokens/total": 2000352, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 27734} +{"epoch": 0.2421875, "grad_norm": 0.35428160429000854, "learning_rate": 9.879197311676887e-05, "loss": 0.005786815658211708, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.0058, "step": 62, "tokens/total": 2032616, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 28217} +{"epoch": 0.24609375, "grad_norm": 0.4580720365047455, "learning_rate": 9.872425169224113e-05, "loss": 0.016534987837076187, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.01667, "step": 63, "tokens/total": 2065968, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 28696} +{"epoch": 0.25, "grad_norm": 0.8986619710922241, "learning_rate": 9.865471072312528e-05, "loss": 0.015300999395549297, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01542, "step": 64, "tokens/total": 2096760, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 29144} +{"epoch": 0.25390625, "grad_norm": 0.7241554856300354, "learning_rate": 9.858335310330492e-05, "loss": 0.013943043537437916, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.01404, "step": 65, "tokens/total": 2124976, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 29569} +{"epoch": 0.2578125, "grad_norm": 0.4668060541152954, "learning_rate": 9.851018180226185e-05, "loss": 0.007569495122879744, "memory/device_reserved (GiB)": 66.37, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.0076, "step": 66, "tokens/total": 2153384, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 30010} +{"epoch": 0.26171875, "grad_norm": 14.654047966003418, "learning_rate": 9.843519986495259e-05, "loss": 0.04696330428123474, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 65.69, "memory/max_allocated (GiB)": 65.69, "ppl": 1.04808, "step": 67, "tokens/total": 2186688, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 30466} +{"epoch": 0.265625, "grad_norm": 0.3339288830757141, "learning_rate": 9.835841041168162e-05, "loss": 0.00433491263538599, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00434, "step": 68, "tokens/total": 2219192, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 30943} +{"epoch": 0.26953125, "grad_norm": 0.30771827697753906, "learning_rate": 9.82798166379715e-05, "loss": 0.003940401133149862, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.00395, "step": 69, "tokens/total": 2249048, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 31364} +{"epoch": 0.2734375, "grad_norm": 0.756891667842865, "learning_rate": 9.819942181443002e-05, "loss": 0.007348169106990099, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00738, "step": 70, "tokens/total": 2280936, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 31782} +{"epoch": 0.27734375, "grad_norm": 1.7235145568847656, "learning_rate": 9.811722928661392e-05, "loss": 0.006318643223494291, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.00634, "step": 71, "tokens/total": 2312920, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 32257} +{"epoch": 0.28125, "grad_norm": 1.360158085823059, "learning_rate": 9.803324247488975e-05, "loss": 0.00574817368760705, "memory/device_reserved (GiB)": 66.62, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.00576, "step": 72, "tokens/total": 2345864, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 32707} +{"epoch": 0.28515625, "grad_norm": 2.0021162033081055, "learning_rate": 9.794746487429161e-05, "loss": 0.02226734720170498, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.02252, "step": 73, "tokens/total": 2379328, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 33207} +{"epoch": 0.2890625, "grad_norm": 0.5226318836212158, "learning_rate": 9.785990005437554e-05, "loss": 0.003972429782152176, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00398, "step": 74, "tokens/total": 2409520, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 33646} +{"epoch": 0.29296875, "grad_norm": 4.642646312713623, "learning_rate": 9.777055165907117e-05, "loss": 0.03148127347230911, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.03198, "step": 75, "tokens/total": 2442240, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 34112} +{"epoch": 0.296875, "grad_norm": 1.182519555091858, "learning_rate": 9.767942340652993e-05, "loss": 0.0129481740295887, "memory/device_reserved (GiB)": 66.75, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01303, "step": 76, "tokens/total": 2474616, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 34592} +{"epoch": 0.30078125, "grad_norm": 2.951450824737549, "learning_rate": 9.758651908897035e-05, "loss": 0.03311644494533539, "memory/device_reserved (GiB)": 66.84, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.03367, "step": 77, "tokens/total": 2509184, "tokens/train_per_sec_per_gpu": 10.04, "tokens/trainable": 35038} +{"epoch": 0.3046875, "grad_norm": 1.3375864028930664, "learning_rate": 9.749184257252033e-05, "loss": 0.023323524743318558, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.0236, "step": 78, "tokens/total": 2543744, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 35529} +{"epoch": 0.30859375, "grad_norm": 0.9486696124076843, "learning_rate": 9.739539779705614e-05, "loss": 0.008588794618844986, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00863, "step": 79, "tokens/total": 2577192, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 35960} +{"epoch": 0.3125, "grad_norm": 1.8694806098937988, "learning_rate": 9.729718877603861e-05, "loss": 0.01225403044372797, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.01233, "step": 80, "tokens/total": 2610528, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 36405} +{"epoch": 0.31640625, "grad_norm": 0.2992646098136902, "learning_rate": 9.719721959634592e-05, "loss": 0.0033109511714428663, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00332, "step": 81, "tokens/total": 2641280, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 36852} +{"epoch": 0.3203125, "grad_norm": 2.5731425285339355, "learning_rate": 9.709549441810375e-05, "loss": 0.02024947479367256, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.02046, "step": 82, "tokens/total": 2671824, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 37291} +{"epoch": 0.32421875, "grad_norm": 1.4940332174301147, "learning_rate": 9.699201747451195e-05, "loss": 0.04130171984434128, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.04217, "step": 83, "tokens/total": 2704016, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 37749} +{"epoch": 0.328125, "grad_norm": 1.7950290441513062, "learning_rate": 9.688679307166854e-05, "loss": 0.03764145076274872, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.03836, "step": 84, "tokens/total": 2736504, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 38205} +{"epoch": 0.33203125, "grad_norm": 0.4296111464500427, "learning_rate": 9.677982558839042e-05, "loss": 0.02022712491452694, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.02043, "step": 85, "tokens/total": 2770552, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 38674} +{"epoch": 0.3359375, "grad_norm": 1.3050512075424194, "learning_rate": 9.66711194760312e-05, "loss": 0.04111558198928833, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 64.19, "memory/max_allocated (GiB)": 64.19, "ppl": 1.04197, "step": 86, "tokens/total": 2803304, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 39124} +{"epoch": 0.33984375, "grad_norm": 0.5314860343933105, "learning_rate": 9.656067925829593e-05, "loss": 0.029045261442661285, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.02947, "step": 87, "tokens/total": 2833864, "tokens/train_per_sec_per_gpu": 10.85, "tokens/trainable": 39600} +{"epoch": 0.34375, "grad_norm": 0.5582631230354309, "learning_rate": 9.644850953105288e-05, "loss": 0.025111963972449303, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.02543, "step": 88, "tokens/total": 2864544, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 40068} +{"epoch": 0.34765625, "grad_norm": 0.34756964445114136, "learning_rate": 9.633461496214225e-05, "loss": 0.008477393537759781, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.00851, "step": 89, "tokens/total": 2891600, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 40452} +{"epoch": 0.3515625, "grad_norm": 0.49778708815574646, "learning_rate": 9.621900029118195e-05, "loss": 0.010842783376574516, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.0109, "step": 90, "tokens/total": 2925328, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 40925} +{"epoch": 0.35546875, "grad_norm": 0.5495241284370422, "learning_rate": 9.610167032937036e-05, "loss": 0.011803231202065945, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.01187, "step": 91, "tokens/total": 2957904, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 41374} +{"epoch": 0.359375, "grad_norm": 1.1704943180084229, "learning_rate": 9.598262995928611e-05, "loss": 0.00900605134665966, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00905, "step": 92, "tokens/total": 2989496, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 41807} +{"epoch": 0.36328125, "grad_norm": 0.3253198564052582, "learning_rate": 9.586188413468492e-05, "loss": 0.008875825442373753, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00892, "step": 93, "tokens/total": 3021704, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 42236} +{"epoch": 0.3671875, "grad_norm": 3.5836570262908936, "learning_rate": 9.57394378802934e-05, "loss": 0.022464338690042496, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.63, "memory/max_allocated (GiB)": 65.63, "ppl": 1.02272, "step": 94, "tokens/total": 3055864, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 42712} +{"epoch": 0.37109375, "grad_norm": 3.1697633266448975, "learning_rate": 9.56152962916e-05, "loss": 0.01695365272462368, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.77, "memory/max_allocated (GiB)": 64.77, "ppl": 1.0171, "step": 95, "tokens/total": 3089592, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 43213} +{"epoch": 0.375, "grad_norm": 0.327038049697876, "learning_rate": 9.548946453464296e-05, "loss": 0.0027528780046850443, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00276, "step": 96, "tokens/total": 3124216, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 43702} +{"epoch": 0.37890625, "grad_norm": 0.34040582180023193, "learning_rate": 9.53619478457953e-05, "loss": 0.003343451302498579, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.00335, "step": 97, "tokens/total": 3157360, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 44158} +{"epoch": 0.3828125, "grad_norm": 1.373887062072754, "learning_rate": 9.523275153154695e-05, "loss": 0.014400104060769081, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.55, "memory/max_allocated (GiB)": 64.55, "ppl": 1.0145, "step": 98, "tokens/total": 3188416, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 44615} +{"epoch": 0.38671875, "grad_norm": 0.3510592579841614, "learning_rate": 9.51018809682839e-05, "loss": 0.0022023911587893963, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.11, "memory/max_allocated (GiB)": 63.11, "ppl": 1.0022, "step": 99, "tokens/total": 3214280, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 45042} +{"epoch": 0.390625, "grad_norm": 1.4889733791351318, "learning_rate": 9.49693416020645e-05, "loss": 0.0378810353577137, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.03861, "step": 100, "tokens/total": 3248168, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 45508} +{"epoch": 0.39453125, "grad_norm": 1.8914262056350708, "learning_rate": 9.483513894839276e-05, "loss": 0.028038017451763153, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.02843, "step": 101, "tokens/total": 3280240, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 45989} +{"epoch": 0.3984375, "grad_norm": 0.3355434536933899, "learning_rate": 9.469927859198888e-05, "loss": 0.0030117651913315058, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00302, "step": 102, "tokens/total": 3311424, "tokens/train_per_sec_per_gpu": 10.01, "tokens/trainable": 46425} +{"epoch": 0.40234375, "grad_norm": 0.07866115123033524, "learning_rate": 9.456176618655689e-05, "loss": 0.0007242051069624722, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00072, "step": 103, "tokens/total": 3345632, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 46876} +{"epoch": 0.40625, "grad_norm": 0.4055706858634949, "learning_rate": 9.442260745454927e-05, "loss": 0.009981175884604454, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.46, "memory/max_allocated (GiB)": 65.46, "ppl": 1.01003, "step": 104, "tokens/total": 3379208, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 47378} +{"epoch": 0.41015625, "grad_norm": 2.1821820735931396, "learning_rate": 9.428180818692884e-05, "loss": 0.02323795109987259, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.02351, "step": 105, "tokens/total": 3409760, "tokens/train_per_sec_per_gpu": 10.63, "tokens/trainable": 47852} +{"epoch": 0.4140625, "grad_norm": 0.7206502556800842, "learning_rate": 9.413937424292791e-05, "loss": 0.02175571210682392, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.02199, "step": 106, "tokens/total": 3441168, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 48325} +{"epoch": 0.41796875, "grad_norm": 4.481708526611328, "learning_rate": 9.399531154980424e-05, "loss": 0.009588333778083324, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00963, "step": 107, "tokens/total": 3474904, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 48818} +{"epoch": 0.421875, "grad_norm": 0.5117279887199402, "learning_rate": 9.384962610259455e-05, "loss": 0.008500827476382256, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.54, "memory/max_allocated (GiB)": 65.54, "ppl": 1.00854, "step": 108, "tokens/total": 3507856, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 49256} +{"epoch": 0.42578125, "grad_norm": 1.4190621376037598, "learning_rate": 9.370232396386494e-05, "loss": 0.017960820347070694, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.01812, "step": 109, "tokens/total": 3542296, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 49701} +{"epoch": 0.4296875, "grad_norm": 19.260887145996094, "learning_rate": 9.355341126345868e-05, "loss": 0.008879575878381729, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00892, "step": 110, "tokens/total": 3577224, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 50192} +{"epoch": 0.43359375, "grad_norm": 0.158192977309227, "learning_rate": 9.340289419824107e-05, "loss": 0.0036595971323549747, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00367, "step": 111, "tokens/total": 3608136, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 50608} +{"epoch": 0.4375, "grad_norm": 0.5744984745979309, "learning_rate": 9.325077903184159e-05, "loss": 0.011053279042243958, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.01111, "step": 112, "tokens/total": 3637736, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 51032} +{"epoch": 0.44140625, "grad_norm": 0.34759321808815, "learning_rate": 9.30970720943932e-05, "loss": 0.0047456044703722, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00476, "step": 113, "tokens/total": 3671112, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 51537} +{"epoch": 0.4453125, "grad_norm": 2.3418729305267334, "learning_rate": 9.2941779782269e-05, "loss": 0.022475510835647583, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.87, "memory/max_allocated (GiB)": 63.87, "ppl": 1.02273, "step": 114, "tokens/total": 3701080, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 52004} +{"epoch": 0.44921875, "grad_norm": 1.1008318662643433, "learning_rate": 9.278490855781596e-05, "loss": 0.021720169112086296, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.02196, "step": 115, "tokens/total": 3734416, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 52487} +{"epoch": 0.453125, "grad_norm": 0.06464647501707077, "learning_rate": 9.262646494908604e-05, "loss": 0.0008140103309415281, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00081, "step": 116, "tokens/total": 3770664, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 52965} +{"epoch": 0.45703125, "grad_norm": 0.6511296629905701, "learning_rate": 9.246645554956457e-05, "loss": 0.005292797461152077, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00531, "step": 117, "tokens/total": 3803832, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 53450} +{"epoch": 0.4609375, "grad_norm": 4.3434648513793945, "learning_rate": 9.230488701789578e-05, "loss": 0.009024572558701038, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00907, "step": 118, "tokens/total": 3837248, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 53927} +{"epoch": 0.46484375, "grad_norm": 1.254514455795288, "learning_rate": 9.214176607760577e-05, "loss": 0.0053986054845154285, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00541, "step": 119, "tokens/total": 3871512, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 54395} +{"epoch": 0.46875, "grad_norm": 0.4203970730304718, "learning_rate": 9.197709951682268e-05, "loss": 0.010416844859719276, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.01047, "step": 120, "tokens/total": 3907056, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 54871} +{"epoch": 0.47265625, "grad_norm": 0.6457905769348145, "learning_rate": 9.181089418799428e-05, "loss": 0.007338094525039196, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00737, "step": 121, "tokens/total": 3938872, "tokens/train_per_sec_per_gpu": 10.31, "tokens/trainable": 55356} +{"epoch": 0.4765625, "grad_norm": 0.2991635203361511, "learning_rate": 9.164315700760271e-05, "loss": 0.004146387334913015, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.00415, "step": 122, "tokens/total": 3971928, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 55836} +{"epoch": 0.48046875, "grad_norm": 0.2760188579559326, "learning_rate": 9.147389495587671e-05, "loss": 0.011546269990503788, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.87, "memory/max_allocated (GiB)": 63.87, "ppl": 1.01161, "step": 123, "tokens/total": 4000432, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 56275} +{"epoch": 0.484375, "grad_norm": 0.23729203641414642, "learning_rate": 9.130311507650116e-05, "loss": 0.001868274062871933, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00187, "step": 124, "tokens/total": 4032984, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 56745} +{"epoch": 0.48828125, "grad_norm": 0.9289960861206055, "learning_rate": 9.113082447632394e-05, "loss": 0.023492245003581047, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.29, "memory/max_allocated (GiB)": 65.29, "ppl": 1.02377, "step": 125, "tokens/total": 4067024, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 57171} +{"epoch": 0.4921875, "grad_norm": 0.1454138606786728, "learning_rate": 9.09570303250602e-05, "loss": 0.0020346660166978836, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00204, "step": 126, "tokens/total": 4098808, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 57642} +{"epoch": 0.49609375, "grad_norm": 0.0497916042804718, "learning_rate": 9.078173985499394e-05, "loss": 0.0004963343380950391, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.0005, "step": 127, "tokens/total": 4131768, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 58134} +{"epoch": 0.5, "grad_norm": 0.1894756257534027, "learning_rate": 9.060496036067713e-05, "loss": 0.0009184989612549543, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00092, "step": 128, "tokens/total": 4164176, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 58564} +{"epoch": 0.50390625, "grad_norm": 0.66973876953125, "learning_rate": 9.042669919862615e-05, "loss": 0.008156592026352882, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.69, "memory/max_allocated (GiB)": 64.69, "ppl": 1.00819, "step": 129, "tokens/total": 4195608, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 59038} +{"epoch": 0.5078125, "grad_norm": 0.06564079225063324, "learning_rate": 9.024696378701557e-05, "loss": 0.0007754103280603886, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00078, "step": 130, "tokens/total": 4227600, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 59495} +{"epoch": 0.51171875, "grad_norm": 0.05212785676121712, "learning_rate": 9.006576160536948e-05, "loss": 0.0007328721694648266, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00073, "step": 131, "tokens/total": 4257936, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 59911} +{"epoch": 0.515625, "grad_norm": 0.43681859970092773, "learning_rate": 8.988310019425035e-05, "loss": 0.004945871885865927, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00496, "step": 132, "tokens/total": 4288360, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 60371} +{"epoch": 0.51953125, "grad_norm": 0.6282080411911011, "learning_rate": 8.969898715494506e-05, "loss": 0.005518488585948944, "memory/device_reserved (GiB)": 65.79, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00553, "step": 133, "tokens/total": 4321224, "tokens/train_per_sec_per_gpu": 10.05, "tokens/trainable": 60827} +{"epoch": 0.5234375, "grad_norm": 0.8081277012825012, "learning_rate": 8.951343014914869e-05, "loss": 0.009909325279295444, "memory/device_reserved (GiB)": 65.93, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00996, "step": 134, "tokens/total": 4354544, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 61320} +{"epoch": 0.52734375, "grad_norm": 0.8799275159835815, "learning_rate": 8.932643689864568e-05, "loss": 0.003485696855932474, "memory/device_reserved (GiB)": 65.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.00349, "step": 135, "tokens/total": 4386120, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 61775} +{"epoch": 0.53125, "grad_norm": 0.6296384334564209, "learning_rate": 8.913801518498845e-05, "loss": 0.016805658116936684, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01695, "step": 136, "tokens/total": 4420072, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 62248} +{"epoch": 0.53515625, "grad_norm": 1.4090408086776733, "learning_rate": 8.894817284917364e-05, "loss": 0.021664908155798912, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.0219, "step": 137, "tokens/total": 4456800, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 62730} +{"epoch": 0.5390625, "grad_norm": 1.4176254272460938, "learning_rate": 8.875691779131569e-05, "loss": 0.005249415524303913, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00526, "step": 138, "tokens/total": 4493688, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 63166} +{"epoch": 0.54296875, "grad_norm": 1.1681898832321167, "learning_rate": 8.856425797031829e-05, "loss": 0.025336138904094696, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.02566, "step": 139, "tokens/total": 4524896, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 63631} +{"epoch": 0.546875, "grad_norm": 1.0585488080978394, "learning_rate": 8.837020140354295e-05, "loss": 0.016859058290719986, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.017, "step": 140, "tokens/total": 4558704, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 64080} +{"epoch": 0.55078125, "grad_norm": 0.26517489552497864, "learning_rate": 8.817475616647554e-05, "loss": 0.0033393127378076315, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.00334, "step": 141, "tokens/total": 4590824, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 64537} +{"epoch": 0.5546875, "grad_norm": 0.49778664112091064, "learning_rate": 8.797793039239017e-05, "loss": 0.006325350608676672, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00635, "step": 142, "tokens/total": 4622328, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 65011} +{"epoch": 0.55859375, "grad_norm": 0.48908644914627075, "learning_rate": 8.777973227201069e-05, "loss": 0.005847309250384569, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.7, "memory/max_allocated (GiB)": 64.7, "ppl": 1.00586, "step": 143, "tokens/total": 4654840, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 65511} +{"epoch": 0.5625, "grad_norm": 0.19024556875228882, "learning_rate": 8.758017005316988e-05, "loss": 0.0019526863470673561, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00195, "step": 144, "tokens/total": 4688200, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 65996} +{"epoch": 0.56640625, "grad_norm": 0.8351379632949829, "learning_rate": 8.737925204046629e-05, "loss": 0.017409879714250565, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.01756, "step": 145, "tokens/total": 4719400, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 66423} +{"epoch": 0.5703125, "grad_norm": 0.40795159339904785, "learning_rate": 8.717698659491851e-05, "loss": 0.006546269170939922, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00657, "step": 146, "tokens/total": 4753928, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 66905} +{"epoch": 0.57421875, "grad_norm": 1.2967678308486938, "learning_rate": 8.697338213361735e-05, "loss": 0.017453521490097046, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01761, "step": 147, "tokens/total": 4788456, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 67363} +{"epoch": 0.578125, "grad_norm": 0.18569128215312958, "learning_rate": 8.676844712937552e-05, "loss": 0.0022632000036537647, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.62, "memory/max_allocated (GiB)": 64.62, "ppl": 1.00227, "step": 148, "tokens/total": 4820096, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 67829} +{"epoch": 0.58203125, "grad_norm": 0.4054660499095917, "learning_rate": 8.656219011037509e-05, "loss": 0.00976892001926899, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00982, "step": 149, "tokens/total": 4850208, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 68286} +{"epoch": 0.5859375, "grad_norm": 0.4472199082374573, "learning_rate": 8.63546196598125e-05, "loss": 0.008995065465569496, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00904, "step": 150, "tokens/total": 4884720, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 68750} +{"epoch": 0.58984375, "grad_norm": 0.7189611196517944, "learning_rate": 8.614574441554145e-05, "loss": 0.0375966913998127, "memory/device_reserved (GiB)": 66.83, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.03831, "step": 151, "tokens/total": 4922000, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 69231} +{"epoch": 0.59375, "grad_norm": 1.3792847394943237, "learning_rate": 8.593557306971349e-05, "loss": 0.00620664656162262, "memory/device_reserved (GiB)": 66.83, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00623, "step": 152, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 69711} +{"epoch": 0.59765625, "grad_norm": 1.061086893081665, "learning_rate": 8.572411436841618e-05, "loss": 0.008188198320567608, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00822, "step": 153, "tokens/total": 4992880, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 70178} +{"epoch": 0.6015625, "grad_norm": 0.1844455599784851, "learning_rate": 8.551137711130922e-05, "loss": 0.003238286590203643, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00324, "step": 154, "tokens/total": 5025136, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 70670} +{"epoch": 0.60546875, "grad_norm": 0.45666101574897766, "learning_rate": 8.529737015125824e-05, "loss": 0.005914853885769844, "memory/device_reserved (GiB)": 67.04, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00593, "step": 155, "tokens/total": 5058912, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 71111} +{"epoch": 0.609375, "grad_norm": 0.4480459988117218, "learning_rate": 8.508210239396639e-05, "loss": 0.007628936320543289, "memory/device_reserved (GiB)": 67.04, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.00766, "step": 156, "tokens/total": 5091432, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 71623} +{"epoch": 0.61328125, "grad_norm": 0.26184138655662537, "learning_rate": 8.486558279760375e-05, "loss": 0.004909180104732513, "memory/device_reserved (GiB)": 67.04, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00492, "step": 157, "tokens/total": 5125920, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 72100} +{"epoch": 0.6171875, "grad_norm": 0.22306346893310547, "learning_rate": 8.464782037243449e-05, "loss": 0.007508713286370039, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00754, "step": 158, "tokens/total": 5159664, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 72598} +{"epoch": 0.62109375, "grad_norm": 0.851625919342041, "learning_rate": 8.442882418044202e-05, "loss": 0.015513923019170761, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.01563, "step": 159, "tokens/total": 5195584, "tokens/train_per_sec_per_gpu": 6.81, "tokens/trainable": 73071} +{"epoch": 0.625, "grad_norm": 0.5214640498161316, "learning_rate": 8.420860333495179e-05, "loss": 0.010544238612055779, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.0106, "step": 160, "tokens/total": 5229664, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 73560} +{"epoch": 0.62890625, "grad_norm": 0.5705488920211792, "learning_rate": 8.398716700025208e-05, "loss": 0.008658247999846935, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.0087, "step": 161, "tokens/total": 5264008, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 74083} +{"epoch": 0.6328125, "grad_norm": 0.8842924237251282, "learning_rate": 8.376452439121266e-05, "loss": 0.02047862485051155, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.02069, "step": 162, "tokens/total": 5294720, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 74530} +{"epoch": 0.63671875, "grad_norm": 2.154099225997925, "learning_rate": 8.354068477290124e-05, "loss": 0.017489410936832428, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.01764, "step": 163, "tokens/total": 5325376, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 74999} +{"epoch": 0.640625, "grad_norm": 0.8750657439231873, "learning_rate": 8.331565746019807e-05, "loss": 0.01625247672200203, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.01639, "step": 164, "tokens/total": 5360240, "tokens/train_per_sec_per_gpu": 7.02, "tokens/trainable": 75450} +{"epoch": 0.64453125, "grad_norm": 1.2757186889648438, "learning_rate": 8.308945181740812e-05, "loss": 0.009979045018553734, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01003, "step": 165, "tokens/total": 5392376, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 75921} +{"epoch": 0.6484375, "grad_norm": 0.5575040578842163, "learning_rate": 8.286207725787153e-05, "loss": 0.009430262260138988, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00947, "step": 166, "tokens/total": 5427456, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 76425} +{"epoch": 0.65234375, "grad_norm": 1.9430668354034424, "learning_rate": 8.263354324357182e-05, "loss": 0.023243257775902748, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.02352, "step": 167, "tokens/total": 5457688, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 76863} +{"epoch": 0.65625, "grad_norm": 0.24351882934570312, "learning_rate": 8.240385928474219e-05, "loss": 0.004574734717607498, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00459, "step": 168, "tokens/total": 5488600, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 77298} +{"epoch": 0.66015625, "grad_norm": 0.4762203097343445, "learning_rate": 8.217303493946967e-05, "loss": 0.008690782822668552, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00873, "step": 169, "tokens/total": 5521856, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 77735} +{"epoch": 0.6640625, "grad_norm": 0.23386144638061523, "learning_rate": 8.194107981329746e-05, "loss": 0.0028458284214138985, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00285, "step": 170, "tokens/total": 5553272, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 78179} +{"epoch": 0.66796875, "grad_norm": 0.24988426268100739, "learning_rate": 8.170800355882518e-05, "loss": 0.0030310875736176968, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.57, "memory/max_allocated (GiB)": 65.57, "ppl": 1.00304, "step": 171, "tokens/total": 5587776, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 78660} +{"epoch": 0.671875, "grad_norm": 0.28632763028144836, "learning_rate": 8.147381587530713e-05, "loss": 0.002610432216897607, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00261, "step": 172, "tokens/total": 5620864, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 79110} +{"epoch": 0.67578125, "grad_norm": 0.5059157609939575, "learning_rate": 8.123852650824877e-05, "loss": 0.005255138501524925, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00527, "step": 173, "tokens/total": 5653656, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 79576} +{"epoch": 0.6796875, "grad_norm": 0.7746137380599976, "learning_rate": 8.100214524900103e-05, "loss": 0.024342037737369537, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.02464, "step": 174, "tokens/total": 5682312, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 80001} +{"epoch": 0.68359375, "grad_norm": 0.4075413644313812, "learning_rate": 8.076468193435301e-05, "loss": 0.003655632957816124, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00366, "step": 175, "tokens/total": 5713520, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 80452} +{"epoch": 0.6875, "grad_norm": 0.8009658455848694, "learning_rate": 8.052614644612253e-05, "loss": 0.008204670622944832, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.00824, "step": 176, "tokens/total": 5745296, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 80886} +{"epoch": 0.69140625, "grad_norm": 0.6779338717460632, "learning_rate": 8.028654871074489e-05, "loss": 0.02953031100332737, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.02997, "step": 177, "tokens/total": 5774336, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 81325} +{"epoch": 0.6953125, "grad_norm": 0.5894985795021057, "learning_rate": 8.004589869885986e-05, "loss": 0.0016039976617321372, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00161, "step": 178, "tokens/total": 5805032, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 81732} +{"epoch": 0.69921875, "grad_norm": 0.27596861124038696, "learning_rate": 7.980420642489674e-05, "loss": 0.0007121656090021133, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00071, "step": 179, "tokens/total": 5832616, "tokens/train_per_sec_per_gpu": 11.15, "tokens/trainable": 82171} +{"epoch": 0.703125, "grad_norm": 0.8324422240257263, "learning_rate": 7.95614819466576e-05, "loss": 0.0069751618430018425, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.007, "step": 180, "tokens/total": 5865568, "tokens/train_per_sec_per_gpu": 10.49, "tokens/trainable": 82670} +{"epoch": 0.70703125, "grad_norm": 1.0424516201019287, "learning_rate": 7.931773536489872e-05, "loss": 0.019223475828766823, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.01941, "step": 181, "tokens/total": 5894256, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 83084} +{"epoch": 0.7109375, "grad_norm": 0.27683040499687195, "learning_rate": 7.907297682291035e-05, "loss": 0.0025163874961435795, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00252, "step": 182, "tokens/total": 5924304, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 83500} +{"epoch": 0.71484375, "grad_norm": 0.3812861442565918, "learning_rate": 7.882721650609442e-05, "loss": 0.0037288980092853308, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00374, "step": 183, "tokens/total": 5955128, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 83896} +{"epoch": 0.71875, "grad_norm": 1.0555505752563477, "learning_rate": 7.85804646415409e-05, "loss": 0.020412011072039604, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.86, "memory/max_allocated (GiB)": 64.86, "ppl": 1.02062, "step": 184, "tokens/total": 5988056, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 84399} +{"epoch": 0.72265625, "grad_norm": 0.03542892634868622, "learning_rate": 7.833273149760207e-05, "loss": 0.0004954856121912599, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.0005, "step": 185, "tokens/total": 6019544, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 84873} +{"epoch": 0.7265625, "grad_norm": 0.7884024977684021, "learning_rate": 7.808402738346527e-05, "loss": 0.0064213573932647705, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00644, "step": 186, "tokens/total": 6051616, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 85296} +{"epoch": 0.73046875, "grad_norm": 0.47317156195640564, "learning_rate": 7.783436264872382e-05, "loss": 0.01341664046049118, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.54, "memory/max_allocated (GiB)": 64.54, "ppl": 1.01351, "step": 187, "tokens/total": 6084304, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 85741} +{"epoch": 0.734375, "grad_norm": 0.3185521960258484, "learning_rate": 7.758374768294647e-05, "loss": 0.005550716537982225, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00557, "step": 188, "tokens/total": 6117896, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 86176} +{"epoch": 0.73828125, "grad_norm": 0.18606840074062347, "learning_rate": 7.733219291524489e-05, "loss": 0.003626073943451047, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00363, "step": 189, "tokens/total": 6152296, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 86649} +{"epoch": 0.7421875, "grad_norm": 0.45820334553718567, "learning_rate": 7.707970881383977e-05, "loss": 0.005606480408459902, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00562, "step": 190, "tokens/total": 6186056, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 87084} +{"epoch": 0.74609375, "grad_norm": 0.8598858714103699, "learning_rate": 7.682630588562518e-05, "loss": 0.011405868455767632, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.01147, "step": 191, "tokens/total": 6219160, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 87499} +{"epoch": 0.75, "grad_norm": 0.2607637643814087, "learning_rate": 7.657199467573129e-05, "loss": 0.004375601187348366, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00439, "step": 192, "tokens/total": 6254840, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 87987} +{"epoch": 0.75390625, "grad_norm": 0.42017000913619995, "learning_rate": 7.631678576708561e-05, "loss": 0.005049226805567741, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00506, "step": 193, "tokens/total": 6289656, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 88470} +{"epoch": 0.7578125, "grad_norm": 0.5039415955543518, "learning_rate": 7.606068977997255e-05, "loss": 0.011034568771719933, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.0111, "step": 194, "tokens/total": 6321776, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 88923} +{"epoch": 0.76171875, "grad_norm": 1.2147960662841797, "learning_rate": 7.580371737159148e-05, "loss": 0.005496613681316376, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00551, "step": 195, "tokens/total": 6352824, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 89338} +{"epoch": 0.765625, "grad_norm": 0.3139284551143646, "learning_rate": 7.554587923561324e-05, "loss": 0.0039002764970064163, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00391, "step": 196, "tokens/total": 6388496, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 89812} +{"epoch": 0.76953125, "grad_norm": 0.8138980269432068, "learning_rate": 7.528718610173511e-05, "loss": 0.0081607885658741, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00819, "step": 197, "tokens/total": 6420600, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 90252} +{"epoch": 0.7734375, "grad_norm": 0.07216157019138336, "learning_rate": 7.502764873523431e-05, "loss": 0.0005838663782924414, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00058, "step": 198, "tokens/total": 6454616, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 90718} +{"epoch": 0.77734375, "grad_norm": 1.7025113105773926, "learning_rate": 7.476727793652011e-05, "loss": 0.00449520954862237, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00451, "step": 199, "tokens/total": 6485680, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 91194} +{"epoch": 0.78125, "grad_norm": 0.6860668063163757, "learning_rate": 7.450608454068415e-05, "loss": 0.006913688033819199, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00694, "step": 200, "tokens/total": 6518512, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 91649} +{"epoch": 0.78515625, "grad_norm": 0.10259273648262024, "learning_rate": 7.424407941704987e-05, "loss": 0.0011981369461864233, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.0012, "step": 201, "tokens/total": 6548728, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 92078} +{"epoch": 0.7890625, "grad_norm": 0.591633677482605, "learning_rate": 7.398127346871986e-05, "loss": 0.0042440989054739475, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00425, "step": 202, "tokens/total": 6582192, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 92536} +{"epoch": 0.79296875, "grad_norm": 1.4961909055709839, "learning_rate": 7.371767763212238e-05, "loss": 0.0028153080493211746, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.00282, "step": 203, "tokens/total": 6617392, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 93032} +{"epoch": 0.796875, "grad_norm": 0.4826553463935852, "learning_rate": 7.345330287655617e-05, "loss": 0.0060087088495492935, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00603, "step": 204, "tokens/total": 6648904, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 93504} +{"epoch": 0.80078125, "grad_norm": 0.6127563118934631, "learning_rate": 7.31881602037339e-05, "loss": 0.018612302839756012, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.01879, "step": 205, "tokens/total": 6679832, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 93925} +{"epoch": 0.8046875, "grad_norm": 0.1850760132074356, "learning_rate": 7.29222606473245e-05, "loss": 0.0007273740484379232, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00073, "step": 206, "tokens/total": 6710784, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 94365} +{"epoch": 0.80859375, "grad_norm": 1.288277506828308, "learning_rate": 7.265561527249383e-05, "loss": 0.0169665664434433, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.01711, "step": 207, "tokens/total": 6744848, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 94845} +{"epoch": 0.8125, "grad_norm": 0.6297875642776489, "learning_rate": 7.238823517544436e-05, "loss": 0.004756874870508909, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00477, "step": 208, "tokens/total": 6777600, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 95285} +{"epoch": 0.81640625, "grad_norm": 0.025383230298757553, "learning_rate": 7.212013148295333e-05, "loss": 0.00041422524373047054, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00041, "step": 209, "tokens/total": 6811024, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 95758} +{"epoch": 0.8203125, "grad_norm": 0.18071730434894562, "learning_rate": 7.185131535190975e-05, "loss": 0.0009568557143211365, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.15, "memory/max_allocated (GiB)": 66.15, "ppl": 1.00096, "step": 210, "tokens/total": 6845376, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 96192} +{"epoch": 0.82421875, "grad_norm": 1.0175156593322754, "learning_rate": 7.158179796885005e-05, "loss": 0.0075608729384839535, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00759, "step": 211, "tokens/total": 6880776, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 96641} +{"epoch": 0.828125, "grad_norm": 0.42007359862327576, "learning_rate": 7.131159054949273e-05, "loss": 0.011319879442453384, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.01138, "step": 212, "tokens/total": 6916160, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 97160} +{"epoch": 0.83203125, "grad_norm": 0.4768296182155609, "learning_rate": 7.104070433827139e-05, "loss": 0.0019353858660906553, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.54, "memory/max_allocated (GiB)": 63.54, "ppl": 1.00194, "step": 213, "tokens/total": 6945320, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 97611} +{"epoch": 0.8359375, "grad_norm": 0.21812878549098969, "learning_rate": 7.076915060786705e-05, "loss": 0.002707911189645529, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00271, "step": 214, "tokens/total": 6977856, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 98111} +{"epoch": 0.83984375, "grad_norm": 0.28817522525787354, "learning_rate": 7.049694065873882e-05, "loss": 0.0030257664620876312, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.00303, "step": 215, "tokens/total": 7011096, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 98575} +{"epoch": 0.84375, "grad_norm": 0.24030598998069763, "learning_rate": 7.022408581865382e-05, "loss": 0.007375575602054596, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.93, "memory/max_allocated (GiB)": 63.93, "ppl": 1.0074, "step": 216, "tokens/total": 7041408, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 98991} +{"epoch": 0.84765625, "grad_norm": 0.5497611165046692, "learning_rate": 6.99505974422157e-05, "loss": 0.007599608041346073, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.6, "memory/max_allocated (GiB)": 63.6, "ppl": 1.00763, "step": 217, "tokens/total": 7071560, "tokens/train_per_sec_per_gpu": 11.52, "tokens/trainable": 99454} +{"epoch": 0.8515625, "grad_norm": 0.020409399643540382, "learning_rate": 6.967648691039213e-05, "loss": 0.0001844228827394545, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00018, "step": 218, "tokens/total": 7104696, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 99925} +{"epoch": 0.85546875, "grad_norm": 0.7434714436531067, "learning_rate": 6.940176563004123e-05, "loss": 0.00324353133328259, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00325, "step": 219, "tokens/total": 7138504, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 100371} +{"epoch": 0.859375, "grad_norm": 2.028691530227661, "learning_rate": 6.912644503343682e-05, "loss": 0.026901546865701675, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.02727, "step": 220, "tokens/total": 7170992, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 100855} +{"epoch": 0.86328125, "grad_norm": 0.10408627986907959, "learning_rate": 6.885053657779273e-05, "loss": 0.000949486275203526, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00095, "step": 221, "tokens/total": 7201176, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 101253} +{"epoch": 0.8671875, "grad_norm": 0.022708479315042496, "learning_rate": 6.857405174478604e-05, "loss": 0.0002283617213834077, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00023, "step": 222, "tokens/total": 7233528, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 101706} +{"epoch": 0.87109375, "grad_norm": 0.30495402216911316, "learning_rate": 6.82970020400792e-05, "loss": 0.003593805246055126, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.0036, "step": 223, "tokens/total": 7267672, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 102158} +{"epoch": 0.875, "grad_norm": 0.7461671829223633, "learning_rate": 6.801939899284132e-05, "loss": 0.006358943413943052, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 62.86, "memory/max_allocated (GiB)": 62.86, "ppl": 1.00638, "step": 224, "tokens/total": 7295592, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 102580} +{"epoch": 0.87890625, "grad_norm": 0.7648396492004395, "learning_rate": 6.774125415526827e-05, "loss": 0.002741862554103136, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00275, "step": 225, "tokens/total": 7325816, "tokens/train_per_sec_per_gpu": 10.08, "tokens/trainable": 103063} +{"epoch": 0.8828125, "grad_norm": 0.34309497475624084, "learning_rate": 6.746257910210214e-05, "loss": 0.002320606028661132, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00232, "step": 226, "tokens/total": 7360888, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 103510} +{"epoch": 0.88671875, "grad_norm": 0.103010393679142, "learning_rate": 6.718338543014937e-05, "loss": 0.0011218972504138947, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00112, "step": 227, "tokens/total": 7390416, "tokens/train_per_sec_per_gpu": 10.3, "tokens/trainable": 103980} +{"epoch": 0.890625, "grad_norm": 1.9638253450393677, "learning_rate": 6.69036847577983e-05, "loss": 0.008329499512910843, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00836, "step": 228, "tokens/total": 7420480, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 104413} +{"epoch": 0.89453125, "grad_norm": 0.466291606426239, "learning_rate": 6.662348872453553e-05, "loss": 0.0025076691526919603, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00251, "step": 229, "tokens/total": 7448992, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 104845} +{"epoch": 0.8984375, "grad_norm": 0.04070416837930679, "learning_rate": 6.63428089904618e-05, "loss": 0.00039455376099795103, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00039, "step": 230, "tokens/total": 7481744, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 105289} +{"epoch": 0.90234375, "grad_norm": 0.08065260946750641, "learning_rate": 6.60616572358065e-05, "loss": 0.0008316644816659391, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00083, "step": 231, "tokens/total": 7516704, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 105722} +{"epoch": 0.90625, "grad_norm": 0.03298543021082878, "learning_rate": 6.578004516044172e-05, "loss": 0.0002038379170699045, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.19, "memory/max_allocated (GiB)": 64.19, "ppl": 1.0002, "step": 232, "tokens/total": 7548592, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 106213} +{"epoch": 0.91015625, "grad_norm": 0.01803118921816349, "learning_rate": 6.549798448339548e-05, "loss": 0.00010591893806122243, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00011, "step": 233, "tokens/total": 7579288, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 106643} +{"epoch": 0.9140625, "grad_norm": 0.45695608854293823, "learning_rate": 6.521548694236384e-05, "loss": 0.0027854463551193476, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00279, "step": 234, "tokens/total": 7611672, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 107136} +{"epoch": 0.91796875, "grad_norm": 1.1209386587142944, "learning_rate": 6.493256429322259e-05, "loss": 0.011600833386182785, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.01167, "step": 235, "tokens/total": 7646288, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 107588} +{"epoch": 0.921875, "grad_norm": 0.1772838830947876, "learning_rate": 6.464922830953799e-05, "loss": 0.0017158652190119028, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00172, "step": 236, "tokens/total": 7680136, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 108087} +{"epoch": 0.92578125, "grad_norm": 0.24603018164634705, "learning_rate": 6.436549078207688e-05, "loss": 0.0003361402195878327, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.61, "memory/max_allocated (GiB)": 64.61, "ppl": 1.00034, "step": 237, "tokens/total": 7713072, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 108533} +{"epoch": 0.9296875, "grad_norm": 0.003944769501686096, "learning_rate": 6.408136351831592e-05, "loss": 3.609908162616193e-05, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00004, "step": 238, "tokens/total": 7747856, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 108965} +{"epoch": 0.93359375, "grad_norm": 0.028763320297002792, "learning_rate": 6.379685834195036e-05, "loss": 0.0004573984770104289, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00046, "step": 239, "tokens/total": 7780136, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 109416} +{"epoch": 0.9375, "grad_norm": 0.11150258034467697, "learning_rate": 6.351198709240186e-05, "loss": 0.00044087195419706404, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.78, "memory/max_allocated (GiB)": 65.78, "ppl": 1.00044, "step": 240, "tokens/total": 7816344, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 109864} +{"epoch": 0.94140625, "grad_norm": 0.02577330358326435, "learning_rate": 6.32267616243259e-05, "loss": 0.00021828088210895658, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00022, "step": 241, "tokens/total": 7850408, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 110345} +{"epoch": 0.9453125, "grad_norm": 0.23798252642154694, "learning_rate": 6.294119380711849e-05, "loss": 0.0023273604456335306, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00233, "step": 242, "tokens/total": 7881528, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 110794} +{"epoch": 0.94921875, "grad_norm": 0.21487674117088318, "learning_rate": 6.265529552442209e-05, "loss": 0.001300458563491702, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.0013, "step": 243, "tokens/total": 7915712, "tokens/train_per_sec_per_gpu": 10.13, "tokens/trainable": 111250} +{"epoch": 0.953125, "grad_norm": 0.6802648305892944, "learning_rate": 6.236907867363127e-05, "loss": 0.012845991179347038, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.61, "memory/max_allocated (GiB)": 64.61, "ppl": 1.01293, "step": 244, "tokens/total": 7947408, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 111693} +{"epoch": 0.95703125, "grad_norm": 0.05710639804601669, "learning_rate": 6.208255516539749e-05, "loss": 0.00034106464590877295, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.26, "memory/max_allocated (GiB)": 65.26, "ppl": 1.00034, "step": 245, "tokens/total": 7979600, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 112158} +{"epoch": 0.9609375, "grad_norm": 0.4075179398059845, "learning_rate": 6.179573692313344e-05, "loss": 0.006925257388502359, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00695, "step": 246, "tokens/total": 8010848, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 112601} +{"epoch": 0.96484375, "grad_norm": 0.21027016639709473, "learning_rate": 6.150863588251694e-05, "loss": 0.00205343053676188, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.00206, "step": 247, "tokens/total": 8040632, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 113016} +{"epoch": 0.96875, "grad_norm": 0.006861444562673569, "learning_rate": 6.122126399099419e-05, "loss": 4.683159204432741e-05, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00005, "step": 248, "tokens/total": 8072464, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 113476} +{"epoch": 0.97265625, "grad_norm": 0.8651519417762756, "learning_rate": 6.0933633207282615e-05, "loss": 0.0074001881293952465, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.96, "memory/max_allocated (GiB)": 64.96, "ppl": 1.00743, "step": 249, "tokens/total": 8106568, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 113941} +{"epoch": 0.9765625, "grad_norm": 1.7307956218719482, "learning_rate": 6.064575550087316e-05, "loss": 0.017627067863941193, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.01778, "step": 250, "tokens/total": 8138496, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 114434} +{"epoch": 0.98046875, "grad_norm": 0.5387976765632629, "learning_rate": 6.0357642851532245e-05, "loss": 0.01109123881906271, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.01115, "step": 251, "tokens/total": 8168080, "tokens/train_per_sec_per_gpu": 10.39, "tokens/trainable": 114898} +{"epoch": 0.984375, "grad_norm": 0.11763789504766464, "learning_rate": 6.0069307248803294e-05, "loss": 0.00039867873420007527, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.0004, "step": 252, "tokens/total": 8200888, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 115331} +{"epoch": 0.98828125, "grad_norm": 0.08922798931598663, "learning_rate": 5.9780760691507635e-05, "loss": 0.0005627688951790333, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00056, "step": 253, "tokens/total": 8230704, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 115732} +{"epoch": 0.9921875, "grad_norm": 0.3104954659938812, "learning_rate": 5.9492015187245334e-05, "loss": 0.01336340606212616, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.01345, "step": 254, "tokens/total": 8266560, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 116221} +{"epoch": 0.99609375, "grad_norm": 1.3461737632751465, "learning_rate": 5.920308275189541e-05, "loss": 0.024143952876329422, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.19, "memory/max_allocated (GiB)": 65.19, "ppl": 1.02444, "step": 255, "tokens/total": 8300184, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 116684} +{"epoch": 1.0, "grad_norm": 0.1961231827735901, "learning_rate": 5.8913975409115874e-05, "loss": 0.0011419380316510797, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00114, "step": 256, "tokens/total": 8333624, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 117126} +{"epoch": 1.00390625, "grad_norm": 0.06312862038612366, "learning_rate": 5.8624705189843395e-05, "loss": 0.0006952512776479125, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.0007, "step": 257, "tokens/total": 8364976, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 117592} +{"epoch": 1.0078125, "grad_norm": 0.09088659286499023, "learning_rate": 5.833528413179249e-05, "loss": 0.000912207760848105, "memory/device_reserved (GiB)": 66.62, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00091, "step": 258, "tokens/total": 8399568, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 118030} +{"epoch": 1.01171875, "grad_norm": 0.7182297110557556, "learning_rate": 5.80457242789548e-05, "loss": 0.00424483697861433, "memory/device_reserved (GiB)": 66.62, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00425, "step": 259, "tokens/total": 8430616, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 118470} +{"epoch": 1.015625, "grad_norm": 0.1510581374168396, "learning_rate": 5.77560376810977e-05, "loss": 0.007091245148330927, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00712, "step": 260, "tokens/total": 8462696, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 118935} +{"epoch": 1.01953125, "grad_norm": 0.9957525134086609, "learning_rate": 5.7466236393263005e-05, "loss": 0.01015563029795885, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.01021, "step": 261, "tokens/total": 8496152, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 119381} +{"epoch": 1.0234375, "grad_norm": 0.145487979054451, "learning_rate": 5.717633247526522e-05, "loss": 0.0030903038568794727, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.0031, "step": 262, "tokens/total": 8532912, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 119849} +{"epoch": 1.02734375, "grad_norm": 0.28214552998542786, "learning_rate": 5.688633799118971e-05, "loss": 0.0028563260566443205, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.00286, "step": 263, "tokens/total": 8565376, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 120319} +{"epoch": 1.03125, "grad_norm": 0.3929193913936615, "learning_rate": 5.659626500889066e-05, "loss": 0.006503222044557333, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00652, "step": 264, "tokens/total": 8597640, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 120804} +{"epoch": 1.03515625, "grad_norm": 0.037327829748392105, "learning_rate": 5.6306125599488905e-05, "loss": 0.0003933248808607459, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.00039, "step": 265, "tokens/total": 8627312, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 121233} +{"epoch": 1.0390625, "grad_norm": 0.1651623547077179, "learning_rate": 5.601593183686955e-05, "loss": 0.0013239247491583228, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.87, "memory/max_allocated (GiB)": 64.87, "ppl": 1.00132, "step": 266, "tokens/total": 8659208, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 121671} +{"epoch": 1.04296875, "grad_norm": 0.05091061443090439, "learning_rate": 5.572569579717961e-05, "loss": 0.0009870207868516445, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.00099, "step": 267, "tokens/total": 8693088, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 122141} +{"epoch": 1.046875, "grad_norm": 0.06784632802009583, "learning_rate": 5.543542955832538e-05, "loss": 0.0007276880205608904, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00073, "step": 268, "tokens/total": 8725576, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 122556} +{"epoch": 1.05078125, "grad_norm": 0.2763011157512665, "learning_rate": 5.514514519946986e-05, "loss": 0.004080546088516712, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00409, "step": 269, "tokens/total": 8755192, "tokens/train_per_sec_per_gpu": 10.45, "tokens/trainable": 122988} +{"epoch": 1.0546875, "grad_norm": 0.023052798584103584, "learning_rate": 5.485485480053015e-05, "loss": 0.00026828653062693775, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00027, "step": 270, "tokens/total": 8787704, "tokens/train_per_sec_per_gpu": 10.68, "tokens/trainable": 123478} +{"epoch": 1.05859375, "grad_norm": 0.2546643018722534, "learning_rate": 5.4564570441674645e-05, "loss": 0.0018094968982040882, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00181, "step": 271, "tokens/total": 8819768, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 123932} +{"epoch": 1.0625, "grad_norm": 0.24677874147891998, "learning_rate": 5.42743042028204e-05, "loss": 0.0012775547802448273, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00128, "step": 272, "tokens/total": 8853352, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 124428} +{"epoch": 1.06640625, "grad_norm": 0.2235422134399414, "learning_rate": 5.3984068163130464e-05, "loss": 0.0010640423279255629, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00106, "step": 273, "tokens/total": 8886840, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 124866} +{"epoch": 1.0703125, "grad_norm": 0.48444974422454834, "learning_rate": 5.369387440051111e-05, "loss": 0.002477427711710334, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00248, "step": 274, "tokens/total": 8920808, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 125324} +{"epoch": 1.07421875, "grad_norm": 0.17306117713451385, "learning_rate": 5.340373499110935e-05, "loss": 0.0014473494375124574, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00145, "step": 275, "tokens/total": 8955880, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 125813} +{"epoch": 1.078125, "grad_norm": 0.018340781331062317, "learning_rate": 5.3113662008810304e-05, "loss": 0.0001786294305929914, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.00018, "step": 276, "tokens/total": 8989520, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 126252} +{"epoch": 1.08203125, "grad_norm": 0.03496924787759781, "learning_rate": 5.282366752473479e-05, "loss": 0.00028647284489125013, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00029, "step": 277, "tokens/total": 9024680, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 126696} +{"epoch": 1.0859375, "grad_norm": 0.6171507239341736, "learning_rate": 5.2533763606737005e-05, "loss": 0.007356297690421343, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00738, "step": 278, "tokens/total": 9056208, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 127173} +{"epoch": 1.08984375, "grad_norm": 0.44598785042762756, "learning_rate": 5.224396231890232e-05, "loss": 0.002376255812123418, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00238, "step": 279, "tokens/total": 9086376, "tokens/train_per_sec_per_gpu": 10.06, "tokens/trainable": 127637} +{"epoch": 1.09375, "grad_norm": 0.024851178750395775, "learning_rate": 5.195427572104522e-05, "loss": 0.0001498935598647222, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00015, "step": 280, "tokens/total": 9118944, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 128112} +{"epoch": 1.09765625, "grad_norm": 0.0055204713717103004, "learning_rate": 5.166471586820751e-05, "loss": 6.182328797876835e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00006, "step": 281, "tokens/total": 9152920, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 128584} +{"epoch": 1.1015625, "grad_norm": 0.004550806246697903, "learning_rate": 5.1375294810156615e-05, "loss": 3.301836841274053e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00003, "step": 282, "tokens/total": 9185728, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 129051} +{"epoch": 1.10546875, "grad_norm": 0.001308328821323812, "learning_rate": 5.1086024590884144e-05, "loss": 1.2590892765729222e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00001, "step": 283, "tokens/total": 9217984, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 129494} +{"epoch": 1.109375, "grad_norm": 12.88686752319336, "learning_rate": 5.079691724810461e-05, "loss": 0.0008428992005065084, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00084, "step": 284, "tokens/total": 9253160, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 129983} +{"epoch": 1.11328125, "grad_norm": 0.010353188961744308, "learning_rate": 5.0507984812754684e-05, "loss": 0.00015588833775836974, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00016, "step": 285, "tokens/total": 9286136, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 130405} +{"epoch": 1.1171875, "grad_norm": 0.20302745699882507, "learning_rate": 5.021923930849237e-05, "loss": 0.0010246532037854195, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00103, "step": 286, "tokens/total": 9318360, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 130829} +{"epoch": 1.12109375, "grad_norm": 0.011052538640797138, "learning_rate": 4.99306927511967e-05, "loss": 0.00013925955863669515, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00014, "step": 287, "tokens/total": 9350288, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 131287} +{"epoch": 1.125, "grad_norm": 0.03764616698026657, "learning_rate": 4.964235714846775e-05, "loss": 0.00010187387670157477, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.0001, "step": 288, "tokens/total": 9382104, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 131712} +{"epoch": 1.12890625, "grad_norm": 0.12959280610084534, "learning_rate": 4.9354244499126866e-05, "loss": 0.0005584300379268825, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00056, "step": 289, "tokens/total": 9414392, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 132168} +{"epoch": 1.1328125, "grad_norm": 0.03718514367938042, "learning_rate": 4.90663667927174e-05, "loss": 0.00015125720528885722, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00015, "step": 290, "tokens/total": 9447112, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 132626} +{"epoch": 1.13671875, "grad_norm": 0.5309748649597168, "learning_rate": 4.877873600900581e-05, "loss": 0.009932255372405052, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00998, "step": 291, "tokens/total": 9483400, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 133119} +{"epoch": 1.140625, "grad_norm": 0.0005936052766628563, "learning_rate": 4.849136411748306e-05, "loss": 1.024955327011412e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.48, "memory/max_allocated (GiB)": 64.48, "ppl": 1.00001, "step": 292, "tokens/total": 9513896, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 133550} +{"epoch": 1.14453125, "grad_norm": 0.8131287693977356, "learning_rate": 4.8204263076866574e-05, "loss": 0.002935096388682723, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00294, "step": 293, "tokens/total": 9550184, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 133992} +{"epoch": 1.1484375, "grad_norm": 0.019180364906787872, "learning_rate": 4.791744483460251e-05, "loss": 0.00013473909348249435, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00013, "step": 294, "tokens/total": 9583920, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 134467} +{"epoch": 1.15234375, "grad_norm": 0.005343259312212467, "learning_rate": 4.7630921326368736e-05, "loss": 5.441412577056326e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00005, "step": 295, "tokens/total": 9617288, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 134895} +{"epoch": 1.15625, "grad_norm": 0.24920421838760376, "learning_rate": 4.7344704475577916e-05, "loss": 0.0022034035064280033, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 62.94, "memory/max_allocated (GiB)": 62.94, "ppl": 1.00221, "step": 296, "tokens/total": 9645600, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 135321} +{"epoch": 1.16015625, "grad_norm": 0.661632776260376, "learning_rate": 4.705880619288153e-05, "loss": 0.008814108557999134, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00885, "step": 297, "tokens/total": 9679456, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 135765} +{"epoch": 1.1640625, "grad_norm": 0.2033599615097046, "learning_rate": 4.677323837567412e-05, "loss": 0.001307208905927837, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00131, "step": 298, "tokens/total": 9711696, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 136196} +{"epoch": 1.16796875, "grad_norm": 0.006124213337898254, "learning_rate": 4.6488012907598146e-05, "loss": 4.28994680987671e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00004, "step": 299, "tokens/total": 9745792, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 136649} +{"epoch": 1.171875, "grad_norm": 0.13173364102840424, "learning_rate": 4.620314165804964e-05, "loss": 0.0010692684445530176, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00107, "step": 300, "tokens/total": 9780648, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 137128} +{"epoch": 1.17578125, "grad_norm": 0.019288677722215652, "learning_rate": 4.591863648168407e-05, "loss": 9.476611739955842e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00009, "step": 301, "tokens/total": 9813032, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 137585} +{"epoch": 1.1796875, "grad_norm": 0.0982256755232811, "learning_rate": 4.5634509217923135e-05, "loss": 0.0005660544848069549, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00057, "step": 302, "tokens/total": 9845200, "tokens/train_per_sec_per_gpu": 10.15, "tokens/trainable": 138054} +{"epoch": 1.18359375, "grad_norm": 0.8769529461860657, "learning_rate": 4.535077169046201e-05, "loss": 0.0007326365448534489, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.29, "memory/max_allocated (GiB)": 65.29, "ppl": 1.00073, "step": 303, "tokens/total": 9878616, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 138559} +{"epoch": 1.1875, "grad_norm": 0.19867637753486633, "learning_rate": 4.506743570677743e-05, "loss": 0.009420561604201794, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00947, "step": 304, "tokens/total": 9910776, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 139050} +{"epoch": 1.19140625, "grad_norm": 0.07451743632555008, "learning_rate": 4.478451305763618e-05, "loss": 0.0003503845364321023, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00035, "step": 305, "tokens/total": 9945480, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 139525} +{"epoch": 1.1953125, "grad_norm": 0.03865562751889229, "learning_rate": 4.450201551660454e-05, "loss": 0.00024432953796349466, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.00024, "step": 306, "tokens/total": 9975992, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 139969} +{"epoch": 1.19921875, "grad_norm": 0.5316394567489624, "learning_rate": 4.4219954839558276e-05, "loss": 0.011726800352334976, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.0118, "step": 307, "tokens/total": 10006440, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 140443} +{"epoch": 1.203125, "grad_norm": 0.0041088201105594635, "learning_rate": 4.393834276419352e-05, "loss": 5.0659778935369104e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00005, "step": 308, "tokens/total": 10042928, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 140926} +{"epoch": 1.20703125, "grad_norm": 0.009005230851471424, "learning_rate": 4.36571910095382e-05, "loss": 5.1494454964995384e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00005, "step": 309, "tokens/total": 10075184, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 141344} +{"epoch": 1.2109375, "grad_norm": 0.008321667090058327, "learning_rate": 4.337651127546448e-05, "loss": 7.621805707458407e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00008, "step": 310, "tokens/total": 10106656, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 141753} +{"epoch": 1.21484375, "grad_norm": 0.2950473129749298, "learning_rate": 4.3096315242201736e-05, "loss": 0.0009225240792147815, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.82, "memory/max_allocated (GiB)": 63.82, "ppl": 1.00092, "step": 311, "tokens/total": 10138536, "tokens/train_per_sec_per_gpu": 10.18, "tokens/trainable": 142219} +{"epoch": 1.21875, "grad_norm": 0.8150671720504761, "learning_rate": 4.2816614569850635e-05, "loss": 0.004490700550377369, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.0045, "step": 312, "tokens/total": 10165768, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 142617} +{"epoch": 1.22265625, "grad_norm": 0.010223480872809887, "learning_rate": 4.2537420897897864e-05, "loss": 7.668741454835981e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00008, "step": 313, "tokens/total": 10201624, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 143103} +{"epoch": 1.2265625, "grad_norm": 0.030257415026426315, "learning_rate": 4.225874584473174e-05, "loss": 0.00024128472432494164, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00024, "step": 314, "tokens/total": 10233304, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 143535} +{"epoch": 1.23046875, "grad_norm": 0.014452429488301277, "learning_rate": 4.19806010071587e-05, "loss": 0.00011826884292531759, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00012, "step": 315, "tokens/total": 10267904, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 144009} +{"epoch": 1.234375, "grad_norm": 0.609462320804596, "learning_rate": 4.170299795992081e-05, "loss": 0.006407205015420914, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00643, "step": 316, "tokens/total": 10301280, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 144460} +{"epoch": 1.23828125, "grad_norm": 0.012841306626796722, "learning_rate": 4.142594825521398e-05, "loss": 0.00020095166109967977, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.0002, "step": 317, "tokens/total": 10337072, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 144936} +{"epoch": 1.2421875, "grad_norm": 0.002928366418927908, "learning_rate": 4.114946342220728e-05, "loss": 3.954542626161128e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.69, "memory/max_allocated (GiB)": 64.69, "ppl": 1.00004, "step": 318, "tokens/total": 10368688, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 145375} +{"epoch": 1.24609375, "grad_norm": 0.02283732034265995, "learning_rate": 4.087355496656321e-05, "loss": 0.0002454883651807904, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00025, "step": 319, "tokens/total": 10400752, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 145849} +{"epoch": 1.25, "grad_norm": 0.1258351355791092, "learning_rate": 4.05982343699588e-05, "loss": 0.000794452615082264, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00079, "step": 320, "tokens/total": 10432112, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 146318} +{"epoch": 1.25390625, "grad_norm": 0.02094362862408161, "learning_rate": 4.0323513089607876e-05, "loss": 0.0002895815996453166, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00029, "step": 321, "tokens/total": 10465472, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 146821} +{"epoch": 1.2578125, "grad_norm": 0.017507269978523254, "learning_rate": 4.004940255778431e-05, "loss": 7.011178968241438e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00007, "step": 322, "tokens/total": 10496336, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 147285} +{"epoch": 1.26171875, "grad_norm": 0.04493589326739311, "learning_rate": 3.977591418134619e-05, "loss": 0.00042936220415867865, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00043, "step": 323, "tokens/total": 10525864, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 147729} +{"epoch": 1.265625, "grad_norm": 0.6885989308357239, "learning_rate": 3.95030593412612e-05, "loss": 0.011752404272556305, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.01182, "step": 324, "tokens/total": 10559680, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 148169} +{"epoch": 1.26953125, "grad_norm": 0.14466321468353271, "learning_rate": 3.923084939213296e-05, "loss": 0.0008022476104088128, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.0008, "step": 325, "tokens/total": 10595400, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 148630} +{"epoch": 1.2734375, "grad_norm": 0.012546399608254433, "learning_rate": 3.895929566172861e-05, "loss": 8.436875941697508e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00008, "step": 326, "tokens/total": 10626680, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 149063} +{"epoch": 1.27734375, "grad_norm": 0.12302988022565842, "learning_rate": 3.868840945050728e-05, "loss": 0.0010858995374292135, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00109, "step": 327, "tokens/total": 10661784, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 149549} +{"epoch": 1.28125, "grad_norm": 0.447982519865036, "learning_rate": 3.841820203114995e-05, "loss": 0.015811581164598465, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.31, "memory/max_allocated (GiB)": 65.31, "ppl": 1.01594, "step": 328, "tokens/total": 10696184, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 150015} +{"epoch": 1.28515625, "grad_norm": 0.05298285186290741, "learning_rate": 3.814868464809027e-05, "loss": 0.0003968967357650399, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.0004, "step": 329, "tokens/total": 10729376, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 150514} +{"epoch": 1.2890625, "grad_norm": 0.12417400628328323, "learning_rate": 3.787986851704667e-05, "loss": 0.00042020343244075775, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00042, "step": 330, "tokens/total": 10764824, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 151021} +{"epoch": 1.29296875, "grad_norm": 0.043861113488674164, "learning_rate": 3.7611764824555654e-05, "loss": 0.0005325410165823996, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00053, "step": 331, "tokens/total": 10797072, "tokens/train_per_sec_per_gpu": 10.11, "tokens/trainable": 151486} +{"epoch": 1.296875, "grad_norm": 0.06210257485508919, "learning_rate": 3.734438472750619e-05, "loss": 0.0005648708320222795, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00057, "step": 332, "tokens/total": 10829376, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 151956} +{"epoch": 1.30078125, "grad_norm": 0.02373451739549637, "learning_rate": 3.707773935267552e-05, "loss": 0.0003148567338939756, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00031, "step": 333, "tokens/total": 10863512, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 152442} +{"epoch": 1.3046875, "grad_norm": 0.06166023388504982, "learning_rate": 3.68118397962661e-05, "loss": 0.0005787039408460259, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00058, "step": 334, "tokens/total": 10894432, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 152890} +{"epoch": 1.30859375, "grad_norm": 0.035197664052248, "learning_rate": 3.654669712344384e-05, "loss": 0.0007974031614139676, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.34, "memory/max_allocated (GiB)": 63.34, "ppl": 1.0008, "step": 335, "tokens/total": 10924848, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 153368} +{"epoch": 1.3125, "grad_norm": 0.6794548034667969, "learning_rate": 3.628232236787763e-05, "loss": 0.005680926609784365, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.0057, "step": 336, "tokens/total": 10959024, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 153809} +{"epoch": 1.31640625, "grad_norm": 0.044938236474990845, "learning_rate": 3.6018726531280144e-05, "loss": 0.0004487236437853426, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00045, "step": 337, "tokens/total": 10991088, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 154258} +{"epoch": 1.3203125, "grad_norm": 0.030016791075468063, "learning_rate": 3.575592058295017e-05, "loss": 0.0005253363051451743, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00053, "step": 338, "tokens/total": 11023904, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 154704} +{"epoch": 1.32421875, "grad_norm": 0.06488785147666931, "learning_rate": 3.549391545931585e-05, "loss": 0.0008973596850410104, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.0009, "step": 339, "tokens/total": 11057680, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 155157} +{"epoch": 1.328125, "grad_norm": 0.09028787165880203, "learning_rate": 3.5232722063479914e-05, "loss": 0.0005557116237469018, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00056, "step": 340, "tokens/total": 11089800, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 155582} +{"epoch": 1.33203125, "grad_norm": 0.011990520171821117, "learning_rate": 3.49723512647657e-05, "loss": 0.0001334405387751758, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00013, "step": 341, "tokens/total": 11117960, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 156047} +{"epoch": 1.3359375, "grad_norm": 0.15495403110980988, "learning_rate": 3.471281389826491e-05, "loss": 0.0025931119453161955, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.0026, "step": 342, "tokens/total": 11151888, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 156500} +{"epoch": 1.33984375, "grad_norm": 0.09401575475931168, "learning_rate": 3.4454120764386764e-05, "loss": 0.0006283820839598775, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.22, "memory/max_allocated (GiB)": 65.22, "ppl": 1.00063, "step": 343, "tokens/total": 11184568, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 156931} +{"epoch": 1.34375, "grad_norm": 0.20172888040542603, "learning_rate": 3.4196282628408526e-05, "loss": 0.0022026589140295982, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00221, "step": 344, "tokens/total": 11216400, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 157385} +{"epoch": 1.34765625, "grad_norm": 0.1200392097234726, "learning_rate": 3.3939310220027456e-05, "loss": 0.0004874504520557821, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00049, "step": 345, "tokens/total": 11250768, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 157817} +{"epoch": 1.3515625, "grad_norm": 0.018574994057416916, "learning_rate": 3.3683214232914404e-05, "loss": 0.0001698791020317003, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00017, "step": 346, "tokens/total": 11281808, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 158273} +{"epoch": 1.35546875, "grad_norm": 0.02151603251695633, "learning_rate": 3.342800532426873e-05, "loss": 0.0003337739617563784, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00033, "step": 347, "tokens/total": 11313968, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 158728} +{"epoch": 1.359375, "grad_norm": 0.0034455980639904737, "learning_rate": 3.317369411437484e-05, "loss": 4.171274122199975e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00004, "step": 348, "tokens/total": 11347288, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 159168} +{"epoch": 1.36328125, "grad_norm": 0.17862966656684875, "learning_rate": 3.292029118616024e-05, "loss": 0.0005628381040878594, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00056, "step": 349, "tokens/total": 11377128, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 159592} +{"epoch": 1.3671875, "grad_norm": 0.006278208456933498, "learning_rate": 3.266780708475511e-05, "loss": 5.7319623010698706e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00006, "step": 350, "tokens/total": 11409488, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 160093} +{"epoch": 1.37109375, "grad_norm": 0.03345426172018051, "learning_rate": 3.241625231705354e-05, "loss": 0.00043902898323722184, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00044, "step": 351, "tokens/total": 11442040, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 160534} +{"epoch": 1.375, "grad_norm": 0.04416341334581375, "learning_rate": 3.216563735127618e-05, "loss": 0.0004448767867870629, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00044, "step": 352, "tokens/total": 11478312, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 161009} +{"epoch": 1.37890625, "grad_norm": 0.05908482149243355, "learning_rate": 3.191597261653475e-05, "loss": 0.0007541541708633304, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.93, "memory/max_allocated (GiB)": 63.93, "ppl": 1.00075, "step": 353, "tokens/total": 11509008, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 161450} +{"epoch": 1.3828125, "grad_norm": 0.3067145049571991, "learning_rate": 3.166726850239794e-05, "loss": 0.0029398833867162466, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00294, "step": 354, "tokens/total": 11541264, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 161900} +{"epoch": 1.38671875, "grad_norm": 0.004175007808953524, "learning_rate": 3.141953535845912e-05, "loss": 4.537180939223617e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00005, "step": 355, "tokens/total": 11573072, "tokens/train_per_sec_per_gpu": 10.25, "tokens/trainable": 162358} +{"epoch": 1.390625, "grad_norm": 0.03534912317991257, "learning_rate": 3.11727834939056e-05, "loss": 0.00024854266666807234, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00025, "step": 356, "tokens/total": 11605224, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 162845} +{"epoch": 1.39453125, "grad_norm": 0.45632144808769226, "learning_rate": 3.092702317708967e-05, "loss": 0.008426932618021965, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00846, "step": 357, "tokens/total": 11637512, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 163299} +{"epoch": 1.3984375, "grad_norm": 0.4608745276927948, "learning_rate": 3.0682264635101276e-05, "loss": 0.0019186005229130387, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00192, "step": 358, "tokens/total": 11671280, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 163774} +{"epoch": 1.40234375, "grad_norm": 0.034627217799425125, "learning_rate": 3.0438518053342407e-05, "loss": 0.00032705185003578663, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00033, "step": 359, "tokens/total": 11702008, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 164199} +{"epoch": 1.40625, "grad_norm": 0.02399802766740322, "learning_rate": 3.0195793575103266e-05, "loss": 8.24507005745545e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00008, "step": 360, "tokens/total": 11732032, "tokens/train_per_sec_per_gpu": 10.16, "tokens/trainable": 164667} +{"epoch": 1.41015625, "grad_norm": 0.16361674666404724, "learning_rate": 2.9954101301140146e-05, "loss": 0.002459253417328, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00246, "step": 361, "tokens/total": 11767472, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.004866559989750385, "learning_rate": 2.9713451289255123e-05, "loss": 4.990555680706166e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00005, "step": 362, "tokens/total": 11798432, "tokens/train_per_sec_per_gpu": 10.63, "tokens/trainable": 165575} +{"epoch": 1.41796875, "grad_norm": 0.061532121151685715, "learning_rate": 2.9473853553877484e-05, "loss": 0.0002233103587059304, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00022, "step": 363, "tokens/total": 11833144, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 166060} +{"epoch": 1.421875, "grad_norm": 0.0045398203656077385, "learning_rate": 2.9235318065647e-05, "loss": 3.5530756576918066e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.00004, "step": 364, "tokens/total": 11865856, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 166478} +{"epoch": 1.42578125, "grad_norm": 0.06990717351436615, "learning_rate": 2.8997854750998964e-05, "loss": 0.000489903031848371, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00049, "step": 365, "tokens/total": 11900400, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 166941} +{"epoch": 1.4296875, "grad_norm": 0.4309478998184204, "learning_rate": 2.8761473491751258e-05, "loss": 0.003531763795763254, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.49, "memory/max_allocated (GiB)": 63.49, "ppl": 1.00354, "step": 366, "tokens/total": 11931368, "tokens/train_per_sec_per_gpu": 9.88, "tokens/trainable": 167381} +{"epoch": 1.43359375, "grad_norm": 0.2046928107738495, "learning_rate": 2.8526184124692883e-05, "loss": 0.0005517607787624002, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00055, "step": 367, "tokens/total": 11963320, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 167830} +{"epoch": 1.4375, "grad_norm": 0.0025828906800597906, "learning_rate": 2.829199644117484e-05, "loss": 2.876102371374145e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00003, "step": 368, "tokens/total": 11997536, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 168266} +{"epoch": 1.44140625, "grad_norm": 0.03573905676603317, "learning_rate": 2.8058920186702553e-05, "loss": 9.962372860172763e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.0001, "step": 369, "tokens/total": 12029528, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 168716} +{"epoch": 1.4453125, "grad_norm": 0.002942521357908845, "learning_rate": 2.782696506053033e-05, "loss": 4.3846797780133784e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00004, "step": 370, "tokens/total": 12062504, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 169176} +{"epoch": 1.44921875, "grad_norm": 0.04530355706810951, "learning_rate": 2.7596140715257824e-05, "loss": 0.0004120065423194319, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.78, "memory/max_allocated (GiB)": 65.78, "ppl": 1.00041, "step": 371, "tokens/total": 12097376, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 169591} +{"epoch": 1.453125, "grad_norm": 0.5567354559898376, "learning_rate": 2.7366456756428184e-05, "loss": 0.0062404172495007515, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00626, "step": 372, "tokens/total": 12131920, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 170041} +{"epoch": 1.45703125, "grad_norm": 0.03164345398545265, "learning_rate": 2.7137922742128486e-05, "loss": 0.0002901932457461953, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00029, "step": 373, "tokens/total": 12164528, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 170516} +{"epoch": 1.4609375, "grad_norm": 0.017656004056334496, "learning_rate": 2.691054818259188e-05, "loss": 0.00015868386253714561, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.00016, "step": 374, "tokens/total": 12197376, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 170952} +{"epoch": 1.46484375, "grad_norm": 0.011905930936336517, "learning_rate": 2.6684342539801933e-05, "loss": 8.932880155043676e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00009, "step": 375, "tokens/total": 12233104, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 171424} +{"epoch": 1.46875, "grad_norm": 0.03603377193212509, "learning_rate": 2.645931522709877e-05, "loss": 8.373761374969035e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.69, "memory/max_allocated (GiB)": 65.69, "ppl": 1.00008, "step": 376, "tokens/total": 12269224, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 171884} +{"epoch": 1.47265625, "grad_norm": 0.012433026917278767, "learning_rate": 2.6235475608787365e-05, "loss": 7.600958633702248e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00008, "step": 377, "tokens/total": 12302952, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 172404} +{"epoch": 1.4765625, "grad_norm": 0.11701660603284836, "learning_rate": 2.6012832999747916e-05, "loss": 0.0005363240488804877, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00054, "step": 378, "tokens/total": 12338424, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 172871} +{"epoch": 1.48046875, "grad_norm": 0.22770877182483673, "learning_rate": 2.579139666504821e-05, "loss": 0.0017345453379675746, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00174, "step": 379, "tokens/total": 12369144, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 173329} +{"epoch": 1.484375, "grad_norm": 0.14032411575317383, "learning_rate": 2.557117581955798e-05, "loss": 0.000867959694005549, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00087, "step": 380, "tokens/total": 12403352, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 173827} +{"epoch": 1.48828125, "grad_norm": 0.02492486871778965, "learning_rate": 2.5352179627565532e-05, "loss": 0.000124996920931153, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00013, "step": 381, "tokens/total": 12441056, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 174287} +{"epoch": 1.4921875, "grad_norm": 0.6258755326271057, "learning_rate": 2.5134417202396277e-05, "loss": 0.011691870167851448, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.01176, "step": 382, "tokens/total": 12474080, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 174731} +{"epoch": 1.49609375, "grad_norm": 0.02518007904291153, "learning_rate": 2.491789760603361e-05, "loss": 0.00023743390920571983, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00024, "step": 383, "tokens/total": 12508992, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 175183} +{"epoch": 1.5, "grad_norm": 0.10066010802984238, "learning_rate": 2.4702629848741764e-05, "loss": 0.0007603240665048361, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00076, "step": 384, "tokens/total": 12540896, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 175655} +{"epoch": 1.50390625, "grad_norm": 0.3432862460613251, "learning_rate": 2.4488622888690785e-05, "loss": 0.0022766513284295797, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00228, "step": 385, "tokens/total": 12568416, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 176085} +{"epoch": 1.5078125, "grad_norm": 0.03948361426591873, "learning_rate": 2.427588563158384e-05, "loss": 0.0002031995973084122, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.0002, "step": 386, "tokens/total": 12599792, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 176549} +{"epoch": 1.51171875, "grad_norm": 0.02324414625763893, "learning_rate": 2.406442693028651e-05, "loss": 0.00010084125096909702, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0001, "step": 387, "tokens/total": 12631224, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 176995} +{"epoch": 1.515625, "grad_norm": 0.29378339648246765, "learning_rate": 2.3854255584458547e-05, "loss": 0.0007628636085428298, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00076, "step": 388, "tokens/total": 12667240, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 177468} +{"epoch": 1.51953125, "grad_norm": 0.18213972449302673, "learning_rate": 2.3645380340187508e-05, "loss": 0.0008685231441631913, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.54, "memory/max_allocated (GiB)": 65.54, "ppl": 1.00087, "step": 389, "tokens/total": 12700488, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 177961} +{"epoch": 1.5234375, "grad_norm": 0.10879874974489212, "learning_rate": 2.3437809889624914e-05, "loss": 0.0005332411383278668, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.00053, "step": 390, "tokens/total": 12734528, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 178417} +{"epoch": 1.52734375, "grad_norm": 0.12571702897548676, "learning_rate": 2.3231552870624487e-05, "loss": 0.0005266463849693537, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00053, "step": 391, "tokens/total": 12767152, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 178862} +{"epoch": 1.53125, "grad_norm": 0.42010143399238586, "learning_rate": 2.3026617866382657e-05, "loss": 0.002808520570397377, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.00281, "step": 392, "tokens/total": 12798168, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 179320} +{"epoch": 1.53515625, "grad_norm": 0.0619850680232048, "learning_rate": 2.2823013405081507e-05, "loss": 0.000701966171618551, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.0007, "step": 393, "tokens/total": 12828896, "tokens/train_per_sec_per_gpu": 10.04, "tokens/trainable": 179758} +{"epoch": 1.5390625, "grad_norm": 0.0797358825802803, "learning_rate": 2.2620747959533722e-05, "loss": 0.0004133188631385565, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00041, "step": 394, "tokens/total": 12860024, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 180211} +{"epoch": 1.54296875, "grad_norm": 0.9284800887107849, "learning_rate": 2.2419829946830123e-05, "loss": 0.011228926479816437, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01129, "step": 395, "tokens/total": 12891472, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 180637} +{"epoch": 1.546875, "grad_norm": 0.06111132353544235, "learning_rate": 2.2220267727989325e-05, "loss": 0.0002741192001849413, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00027, "step": 396, "tokens/total": 12925648, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 181142} +{"epoch": 1.55078125, "grad_norm": 0.0386623740196228, "learning_rate": 2.202206960760984e-05, "loss": 0.00030511151999235153, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.77, "memory/max_allocated (GiB)": 64.77, "ppl": 1.00031, "step": 397, "tokens/total": 12959200, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 181597} +{"epoch": 1.5546875, "grad_norm": 0.009326125495135784, "learning_rate": 2.182524383352446e-05, "loss": 0.00012094163685105741, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00012, "step": 398, "tokens/total": 12989624, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 182030} +{"epoch": 1.55859375, "grad_norm": 0.004612342920154333, "learning_rate": 2.1629798596457056e-05, "loss": 5.0893748266389593e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 62.97, "memory/max_allocated (GiB)": 62.97, "ppl": 1.00005, "step": 399, "tokens/total": 13019040, "tokens/train_per_sec_per_gpu": 10.48, "tokens/trainable": 182496} +{"epoch": 1.5625, "grad_norm": 0.002366051310673356, "learning_rate": 2.1435742029681725e-05, "loss": 3.788472531596199e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00004, "step": 400, "tokens/total": 13051808, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 182963} +{"epoch": 1.56640625, "grad_norm": 0.07983515411615372, "learning_rate": 2.124308220868431e-05, "loss": 0.0004530831938609481, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00045, "step": 401, "tokens/total": 13085520, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 183416} +{"epoch": 1.5703125, "grad_norm": 0.002844394650310278, "learning_rate": 2.105182715082638e-05, "loss": 2.3692939066677354e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00002, "step": 402, "tokens/total": 13116224, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 183857} +{"epoch": 1.57421875, "grad_norm": 0.017773408442735672, "learning_rate": 2.0861984815011552e-05, "loss": 0.0001713308593025431, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00017, "step": 403, "tokens/total": 13149984, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 184296} +{"epoch": 1.578125, "grad_norm": 0.27903875708580017, "learning_rate": 2.0673563101354323e-05, "loss": 0.0014973131474107504, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.0015, "step": 404, "tokens/total": 13185488, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 184719} +{"epoch": 1.58203125, "grad_norm": 0.007585854735225439, "learning_rate": 2.0486569850851317e-05, "loss": 4.342058673501015e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00004, "step": 405, "tokens/total": 13216320, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 185172} +{"epoch": 1.5859375, "grad_norm": 0.008583984337747097, "learning_rate": 2.0301012845054956e-05, "loss": 0.00013044409570284188, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00013, "step": 406, "tokens/total": 13248136, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 185646} +{"epoch": 1.58984375, "grad_norm": 0.192463681101799, "learning_rate": 2.011689980574966e-05, "loss": 0.001374830026179552, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00138, "step": 407, "tokens/total": 13280640, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 186129} +{"epoch": 1.59375, "grad_norm": 0.4103609323501587, "learning_rate": 1.993423839463052e-05, "loss": 0.0072402311488986015, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00727, "step": 408, "tokens/total": 13314160, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 186610} +{"epoch": 1.59765625, "grad_norm": 0.3786790370941162, "learning_rate": 1.975303621298445e-05, "loss": 0.0009986466029658914, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.51, "memory/max_allocated (GiB)": 63.51, "ppl": 1.001, "step": 409, "tokens/total": 13345024, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 187066} +{"epoch": 1.6015625, "grad_norm": 0.013422933407127857, "learning_rate": 1.957330080137385e-05, "loss": 0.00012089712254237384, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00012, "step": 410, "tokens/total": 13377248, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 187567} +{"epoch": 1.60546875, "grad_norm": 0.013006238266825676, "learning_rate": 1.9395039639322864e-05, "loss": 0.00013458175817504525, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00013, "step": 411, "tokens/total": 13410992, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 188056} +{"epoch": 1.609375, "grad_norm": 0.002013697987422347, "learning_rate": 1.9218260145006073e-05, "loss": 2.769191632978618e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00003, "step": 412, "tokens/total": 13442728, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 188502} +{"epoch": 1.61328125, "grad_norm": 0.033323682844638824, "learning_rate": 1.904296967493982e-05, "loss": 0.00029168184846639633, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00029, "step": 413, "tokens/total": 13474824, "tokens/train_per_sec_per_gpu": 10.22, "tokens/trainable": 188995} +{"epoch": 1.6171875, "grad_norm": 0.25221338868141174, "learning_rate": 1.8869175523676064e-05, "loss": 0.0018457933329045773, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00185, "step": 414, "tokens/total": 13506528, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 189483} +{"epoch": 1.62109375, "grad_norm": 0.06527186185121536, "learning_rate": 1.869688492349885e-05, "loss": 0.00040267728036269546, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.0004, "step": 415, "tokens/total": 13537792, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 189935} +{"epoch": 1.625, "grad_norm": 0.1468157023191452, "learning_rate": 1.85261050441233e-05, "loss": 0.0006921250023879111, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.00069, "step": 416, "tokens/total": 13573136, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 190399} +{"epoch": 1.62890625, "grad_norm": 0.0417628213763237, "learning_rate": 1.8356842992397304e-05, "loss": 0.0003925430355593562, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00039, "step": 417, "tokens/total": 13607240, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 190862} +{"epoch": 1.6328125, "grad_norm": 0.0006051304517313838, "learning_rate": 1.8189105812005714e-05, "loss": 4.9738223424355965e-06, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.0, "step": 418, "tokens/total": 13638920, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 191294} +{"epoch": 1.63671875, "grad_norm": 0.004458671435713768, "learning_rate": 1.802290048317732e-05, "loss": 4.0751518099568784e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00004, "step": 419, "tokens/total": 13672320, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 191764} +{"epoch": 1.640625, "grad_norm": 0.13117268681526184, "learning_rate": 1.785823392239424e-05, "loss": 0.000276081234915182, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.00028, "step": 420, "tokens/total": 13703312, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 192196} +{"epoch": 1.64453125, "grad_norm": 0.008243794552981853, "learning_rate": 1.7695112982104225e-05, "loss": 9.758413943927735e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.0001, "step": 421, "tokens/total": 13734440, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 192612} +{"epoch": 1.6484375, "grad_norm": 0.0028085531666874886, "learning_rate": 1.7533544450435433e-05, "loss": 1.1022128092008643e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.62, "memory/max_allocated (GiB)": 64.62, "ppl": 1.00001, "step": 422, "tokens/total": 13766944, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 193059} +{"epoch": 1.65234375, "grad_norm": 0.12257950752973557, "learning_rate": 1.7373535050913946e-05, "loss": 0.00011871296737808734, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00012, "step": 423, "tokens/total": 13797120, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 193466} +{"epoch": 1.65625, "grad_norm": 0.005482533946633339, "learning_rate": 1.721509144218405e-05, "loss": 5.200921805226244e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00005, "step": 424, "tokens/total": 13828040, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 193910} +{"epoch": 1.66015625, "grad_norm": 0.005994994658976793, "learning_rate": 1.705822021773101e-05, "loss": 4.218163667246699e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.06, "memory/max_allocated (GiB)": 63.06, "ppl": 1.00004, "step": 425, "tokens/total": 13856328, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 194314} +{"epoch": 1.6640625, "grad_norm": 0.12957753241062164, "learning_rate": 1.69029279056068e-05, "loss": 0.00025902624474838376, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00026, "step": 426, "tokens/total": 13891312, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 194782} +{"epoch": 1.66796875, "grad_norm": 0.007826692424714565, "learning_rate": 1.6749220968158415e-05, "loss": 0.00011911232286365703, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00012, "step": 427, "tokens/total": 13925272, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 195254} +{"epoch": 1.671875, "grad_norm": 0.0015283924294635653, "learning_rate": 1.659710580175893e-05, "loss": 2.3820846763555892e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.53, "memory/max_allocated (GiB)": 63.53, "ppl": 1.00002, "step": 428, "tokens/total": 13955752, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 195702} +{"epoch": 1.67578125, "grad_norm": 0.1347159594297409, "learning_rate": 1.644658873654133e-05, "loss": 0.0015075355768203735, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00151, "step": 429, "tokens/total": 13988000, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 196164} +{"epoch": 1.6796875, "grad_norm": 0.03465171158313751, "learning_rate": 1.629767603613508e-05, "loss": 0.0001622816052986309, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00016, "step": 430, "tokens/total": 14018712, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 196604} +{"epoch": 1.68359375, "grad_norm": 0.01177434716373682, "learning_rate": 1.615037389740547e-05, "loss": 0.00010697231482481584, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00011, "step": 431, "tokens/total": 14051992, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 197070} +{"epoch": 1.6875, "grad_norm": 0.12081583589315414, "learning_rate": 1.600468845019576e-05, "loss": 0.0005837082280777395, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.36, "memory/max_allocated (GiB)": 63.36, "ppl": 1.00058, "step": 432, "tokens/total": 14082624, "tokens/train_per_sec_per_gpu": 10.55, "tokens/trainable": 197552} +{"epoch": 1.69140625, "grad_norm": 0.025729060173034668, "learning_rate": 1.5860625757072092e-05, "loss": 0.0001959516084752977, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.0002, "step": 433, "tokens/total": 14113304, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 197990} +{"epoch": 1.6953125, "grad_norm": 0.04353093355894089, "learning_rate": 1.571819181307116e-05, "loss": 0.00022884301142767072, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00023, "step": 434, "tokens/total": 14149592, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 198513} +{"epoch": 1.69921875, "grad_norm": 0.01941860467195511, "learning_rate": 1.557739254545075e-05, "loss": 0.00011502222332637757, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00012, "step": 435, "tokens/total": 14183584, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 198979} +{"epoch": 1.703125, "grad_norm": 0.02035452425479889, "learning_rate": 1.543823381344311e-05, "loss": 7.21164615242742e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00007, "step": 436, "tokens/total": 14213688, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 199443} +{"epoch": 1.70703125, "grad_norm": 0.08621188253164291, "learning_rate": 1.5300721408011114e-05, "loss": 0.0010246328311040998, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.63, "memory/max_allocated (GiB)": 65.63, "ppl": 1.00103, "step": 437, "tokens/total": 14246480, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 199899} +{"epoch": 1.7109375, "grad_norm": 0.010995444841682911, "learning_rate": 1.5164861051607254e-05, "loss": 5.404639523476362e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00005, "step": 438, "tokens/total": 14278320, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 200368} +{"epoch": 1.71484375, "grad_norm": 0.036437515169382095, "learning_rate": 1.5030658397935521e-05, "loss": 0.00010468787513673306, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.0001, "step": 439, "tokens/total": 14313016, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 200819} +{"epoch": 1.71875, "grad_norm": 0.16491706669330597, "learning_rate": 1.4898119031716104e-05, "loss": 0.0008510678308084607, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00085, "step": 440, "tokens/total": 14343352, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 201275} +{"epoch": 1.72265625, "grad_norm": 0.4867701232433319, "learning_rate": 1.476724846845306e-05, "loss": 0.0024803406558930874, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00248, "step": 441, "tokens/total": 14377736, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 201699} +{"epoch": 1.7265625, "grad_norm": 0.03378753736615181, "learning_rate": 1.463805215420471e-05, "loss": 0.00015580846229568124, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00016, "step": 442, "tokens/total": 14415056, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 202200} +{"epoch": 1.73046875, "grad_norm": 0.627608597278595, "learning_rate": 1.451053546535705e-05, "loss": 0.004206348210573196, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00422, "step": 443, "tokens/total": 14449544, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 202640} +{"epoch": 1.734375, "grad_norm": 0.028130073100328445, "learning_rate": 1.438470370840001e-05, "loss": 0.00015085046470630914, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00015, "step": 444, "tokens/total": 14483472, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 203122} +{"epoch": 1.73828125, "grad_norm": 0.21414367854595184, "learning_rate": 1.4260562119706606e-05, "loss": 0.0011659536976367235, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.19, "memory/max_allocated (GiB)": 65.19, "ppl": 1.00117, "step": 445, "tokens/total": 14517648, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 203587} +{"epoch": 1.7421875, "grad_norm": 0.004941586870700121, "learning_rate": 1.413811586531508e-05, "loss": 8.568211342208087e-06, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00001, "step": 446, "tokens/total": 14553056, "tokens/train_per_sec_per_gpu": 6.87, "tokens/trainable": 204024} +{"epoch": 1.74609375, "grad_norm": 0.16588707268238068, "learning_rate": 1.4017370040713884e-05, "loss": 0.0005872166948392987, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.00059, "step": 447, "tokens/total": 14584280, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 204465} +{"epoch": 1.75, "grad_norm": 0.2744143009185791, "learning_rate": 1.3898329670629645e-05, "loss": 0.00013718268019147217, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00014, "step": 448, "tokens/total": 14618320, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 204923} +{"epoch": 1.75390625, "grad_norm": 0.0275246724486351, "learning_rate": 1.3780999708818058e-05, "loss": 0.00010213463974650949, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.0001, "step": 449, "tokens/total": 14651264, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 205388} +{"epoch": 1.7578125, "grad_norm": 0.014769329689443111, "learning_rate": 1.3665385037857758e-05, "loss": 7.873369031585753e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00008, "step": 450, "tokens/total": 14682896, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 205873} +{"epoch": 1.76171875, "grad_norm": 0.2417842596769333, "learning_rate": 1.3551490468947126e-05, "loss": 0.0031464379280805588, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00315, "step": 451, "tokens/total": 14715272, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 206346} +{"epoch": 1.765625, "grad_norm": 0.006371435709297657, "learning_rate": 1.3439320741704075e-05, "loss": 6.626218237215653e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00007, "step": 452, "tokens/total": 14744840, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 206776} +{"epoch": 1.76953125, "grad_norm": 0.0007058748160488904, "learning_rate": 1.3328880523968808e-05, "loss": 8.83955908648204e-06, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00001, "step": 453, "tokens/total": 14778008, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 207233} +{"epoch": 1.7734375, "grad_norm": 0.019202683120965958, "learning_rate": 1.3220174411609587e-05, "loss": 2.2168362193042412e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00002, "step": 454, "tokens/total": 14808472, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 207695} +{"epoch": 1.77734375, "grad_norm": 0.1615113765001297, "learning_rate": 1.3113206928331471e-05, "loss": 0.0009070101659744978, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00091, "step": 455, "tokens/total": 14841048, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 208144} +{"epoch": 1.78125, "grad_norm": 0.0229168888181448, "learning_rate": 1.300798252548806e-05, "loss": 0.00012506514030974358, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00013, "step": 456, "tokens/total": 14876904, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 208602} +{"epoch": 1.78515625, "grad_norm": 0.0027574466075748205, "learning_rate": 1.2904505581896265e-05, "loss": 2.861038774426561e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.58, "memory/max_allocated (GiB)": 64.58, "ppl": 1.00003, "step": 457, "tokens/total": 14909512, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 209040} +{"epoch": 1.7890625, "grad_norm": 0.002107110572978854, "learning_rate": 1.2802780403654082e-05, "loss": 2.4447712348774076e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00002, "step": 458, "tokens/total": 14942048, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 209501} +{"epoch": 1.79296875, "grad_norm": 0.01516713947057724, "learning_rate": 1.2702811223961408e-05, "loss": 8.489088213536888e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.00008, "step": 459, "tokens/total": 14977032, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 209993} +{"epoch": 1.796875, "grad_norm": 0.46463361382484436, "learning_rate": 1.2604602202943861e-05, "loss": 0.000717434159014374, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00072, "step": 460, "tokens/total": 15011904, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 210455} +{"epoch": 1.80078125, "grad_norm": 0.03563816100358963, "learning_rate": 1.2508157427479686e-05, "loss": 0.00023852428421378136, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00024, "step": 461, "tokens/total": 15045504, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 210960} +{"epoch": 1.8046875, "grad_norm": 0.002728453604504466, "learning_rate": 1.2413480911029655e-05, "loss": 1.7666707208263688e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.15, "memory/max_allocated (GiB)": 66.15, "ppl": 1.00002, "step": 462, "tokens/total": 15078536, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 211405} +{"epoch": 1.80859375, "grad_norm": 0.006154930219054222, "learning_rate": 1.2320576593470082e-05, "loss": 4.179458483122289e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00004, "step": 463, "tokens/total": 15111640, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 211881} +{"epoch": 1.8125, "grad_norm": 0.005275407340377569, "learning_rate": 1.2229448340928828e-05, "loss": 3.075327185797505e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00003, "step": 464, "tokens/total": 15142608, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 212338} +{"epoch": 1.81640625, "grad_norm": 0.4170791506767273, "learning_rate": 1.2140099945624458e-05, "loss": 0.00232494343072176, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.00233, "step": 465, "tokens/total": 15175048, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 212803} +{"epoch": 1.8203125, "grad_norm": 0.0016872886335477233, "learning_rate": 1.205253512570841e-05, "loss": 2.0755825971718878e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00002, "step": 466, "tokens/total": 15205776, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 213215} +{"epoch": 1.82421875, "grad_norm": 0.002580332802608609, "learning_rate": 1.1966757525110255e-05, "loss": 3.6648438253905624e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.00004, "step": 467, "tokens/total": 15239240, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 213675} +{"epoch": 1.828125, "grad_norm": 0.262762188911438, "learning_rate": 1.1882770713386095e-05, "loss": 0.0007886799285188317, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00079, "step": 468, "tokens/total": 15273696, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 214169} +{"epoch": 1.83203125, "grad_norm": 0.021419964730739594, "learning_rate": 1.180057818556998e-05, "loss": 9.211141150444746e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00009, "step": 469, "tokens/total": 15307632, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 214624} +{"epoch": 1.8359375, "grad_norm": 0.523200273513794, "learning_rate": 1.1720183362028494e-05, "loss": 0.0017333043506368995, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00173, "step": 470, "tokens/total": 15339960, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 215087} +{"epoch": 1.83984375, "grad_norm": 0.0608169250190258, "learning_rate": 1.1641589588318387e-05, "loss": 0.00045719894114881754, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00046, "step": 471, "tokens/total": 15373640, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 215553} +{"epoch": 1.84375, "grad_norm": 0.0005098847323097289, "learning_rate": 1.1564800135047418e-05, "loss": 9.669560313341208e-06, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00001, "step": 472, "tokens/total": 15406304, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 216018} +{"epoch": 1.84765625, "grad_norm": 0.0017268091905862093, "learning_rate": 1.148981819773816e-05, "loss": 1.9491733837639913e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00002, "step": 473, "tokens/total": 15440128, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 216477} +{"epoch": 1.8515625, "grad_norm": 0.03877732530236244, "learning_rate": 1.1416646896695086e-05, "loss": 0.00017125741578638554, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00017, "step": 474, "tokens/total": 15471600, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 216949} +{"epoch": 1.85546875, "grad_norm": 0.17557291686534882, "learning_rate": 1.1345289276874717e-05, "loss": 0.0011425464181229472, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00114, "step": 475, "tokens/total": 15507016, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 217408} +{"epoch": 1.859375, "grad_norm": 0.04996359348297119, "learning_rate": 1.1275748307758873e-05, "loss": 0.00011885951244039461, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00012, "step": 476, "tokens/total": 15542600, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 217854} +{"epoch": 1.86328125, "grad_norm": 0.10574400424957275, "learning_rate": 1.1208026883231147e-05, "loss": 0.00034784935996867716, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00035, "step": 477, "tokens/total": 15577320, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 218337} +{"epoch": 1.8671875, "grad_norm": 0.015722164884209633, "learning_rate": 1.1142127821456433e-05, "loss": 1.6680180124239996e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00002, "step": 478, "tokens/total": 15608840, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 218788} +{"epoch": 1.87109375, "grad_norm": 0.0038280803710222244, "learning_rate": 1.1078053864763674e-05, "loss": 3.136830491712317e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00003, "step": 479, "tokens/total": 15640160, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 219219} +{"epoch": 1.875, "grad_norm": 0.02965310961008072, "learning_rate": 1.1015807679531756e-05, "loss": 7.465291855623946e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00007, "step": 480, "tokens/total": 15673920, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 219676} +{"epoch": 1.87890625, "grad_norm": 0.45978236198425293, "learning_rate": 1.0955391856078528e-05, "loss": 0.014598238281905651, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.01471, "step": 481, "tokens/total": 15705512, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 220125} +{"epoch": 1.8828125, "grad_norm": 0.21689145267009735, "learning_rate": 1.0896808908553007e-05, "loss": 0.0009163266513496637, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00092, "step": 482, "tokens/total": 15742096, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 220615} +{"epoch": 1.88671875, "grad_norm": 0.0011335723102092743, "learning_rate": 1.0840061274830763e-05, "loss": 1.0636754268489312e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00001, "step": 483, "tokens/total": 15773064, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 221081} +{"epoch": 1.890625, "grad_norm": 0.005387214012444019, "learning_rate": 1.0785151316412473e-05, "loss": 3.622533768066205e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00004, "step": 484, "tokens/total": 15805368, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 221546} +{"epoch": 1.89453125, "grad_norm": 0.0006892283563502133, "learning_rate": 1.0732081318325639e-05, "loss": 8.70491385285277e-06, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.57, "memory/max_allocated (GiB)": 65.57, "ppl": 1.00001, "step": 485, "tokens/total": 15839480, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 222014} +{"epoch": 1.8984375, "grad_norm": 0.002139100106433034, "learning_rate": 1.0680853489029501e-05, "loss": 2.7467322070151567e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00003, "step": 486, "tokens/total": 15870672, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 222446} +{"epoch": 1.90234375, "grad_norm": 0.0006005860050208867, "learning_rate": 1.0631469960323152e-05, "loss": 8.144122148223687e-06, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.00001, "step": 487, "tokens/total": 15902784, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 222899} +{"epoch": 1.90625, "grad_norm": 0.010495632886886597, "learning_rate": 1.0583932787256783e-05, "loss": 6.808060425100848e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00007, "step": 488, "tokens/total": 15935456, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 223333} +{"epoch": 1.91015625, "grad_norm": 0.006051642820239067, "learning_rate": 1.0538243948046206e-05, "loss": 3.292399560450576e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00003, "step": 489, "tokens/total": 15967960, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 223780} +{"epoch": 1.9140625, "grad_norm": 0.35992592573165894, "learning_rate": 1.0494405343990523e-05, "loss": 0.003955667372792959, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00396, "step": 490, "tokens/total": 16003904, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 224253} +{"epoch": 1.91796875, "grad_norm": 0.0012351969489827752, "learning_rate": 1.0452418799392985e-05, "loss": 1.2015878382953815e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00001, "step": 491, "tokens/total": 16035816, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 224699} +{"epoch": 1.921875, "grad_norm": 0.002152453176677227, "learning_rate": 1.0412286061485102e-05, "loss": 2.0569101252476685e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.26, "memory/max_allocated (GiB)": 65.26, "ppl": 1.00002, "step": 492, "tokens/total": 16066880, "tokens/train_per_sec_per_gpu": 10.58, "tokens/trainable": 225147} +{"epoch": 1.92578125, "grad_norm": 0.013635799288749695, "learning_rate": 1.03740088003539e-05, "loss": 8.669827366247773e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00009, "step": 493, "tokens/total": 16101168, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 225596} +{"epoch": 1.9296875, "grad_norm": 0.5157877802848816, "learning_rate": 1.0337588608872463e-05, "loss": 0.003955292049795389, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00396, "step": 494, "tokens/total": 16135208, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 226037} +{"epoch": 1.93359375, "grad_norm": 0.03309299796819687, "learning_rate": 1.0303027002633622e-05, "loss": 0.00015796285879332572, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00016, "step": 495, "tokens/total": 16168296, "tokens/train_per_sec_per_gpu": 10.59, "tokens/trainable": 226503} +{"epoch": 1.9375, "grad_norm": 0.04906133562326431, "learning_rate": 1.0270325419886884e-05, "loss": 0.0001996164210140705, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.0002, "step": 496, "tokens/total": 16203192, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 226953} +{"epoch": 1.94140625, "grad_norm": 0.2317824512720108, "learning_rate": 1.0239485221478599e-05, "loss": 0.0013978660572320223, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.58, "memory/max_allocated (GiB)": 64.58, "ppl": 1.0014, "step": 497, "tokens/total": 16235168, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 227401} +{"epoch": 1.9453125, "grad_norm": 0.012604068033397198, "learning_rate": 1.0210507690795292e-05, "loss": 4.944328247802332e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00005, "step": 498, "tokens/total": 16267576, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 227860} +{"epoch": 1.94921875, "grad_norm": 0.004482830874621868, "learning_rate": 1.0183394033710305e-05, "loss": 2.6836696633836254e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00003, "step": 499, "tokens/total": 16302416, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 228348} +{"epoch": 1.953125, "grad_norm": 0.005735613871365786, "learning_rate": 1.0158145378533583e-05, "loss": 3.9584025216754526e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00004, "step": 500, "tokens/total": 16336088, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 228823} +{"epoch": 1.95703125, "grad_norm": 0.15193478763103485, "learning_rate": 1.0134762775964726e-05, "loss": 0.0010404016356915236, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00104, "step": 501, "tokens/total": 16367096, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 229280} +{"epoch": 1.9609375, "grad_norm": 0.12797370553016663, "learning_rate": 1.0113247199049278e-05, "loss": 0.00110280339140445, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.0011, "step": 502, "tokens/total": 16399824, "tokens/train_per_sec_per_gpu": 10.22, "tokens/trainable": 229758} +{"epoch": 1.96484375, "grad_norm": 0.0004716879047919065, "learning_rate": 1.0093599543138205e-05, "loss": 7.23748053133022e-06, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00001, "step": 503, "tokens/total": 16432392, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 230200} +{"epoch": 1.96875, "grad_norm": 0.0009292601025663316, "learning_rate": 1.0075820625850675e-05, "loss": 9.495295671513304e-06, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00001, "step": 504, "tokens/total": 16463792, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 230668} +{"epoch": 1.97265625, "grad_norm": 0.0013445125659927726, "learning_rate": 1.0059911187040013e-05, "loss": 1.777561556082219e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.48, "memory/max_allocated (GiB)": 64.48, "ppl": 1.00002, "step": 505, "tokens/total": 16492384, "tokens/train_per_sec_per_gpu": 10.16, "tokens/trainable": 231103} +{"epoch": 1.9765625, "grad_norm": 0.0007164510316215456, "learning_rate": 1.0045871888762893e-05, "loss": 8.343638910446316e-06, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00001, "step": 506, "tokens/total": 16526792, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 231543} +{"epoch": 1.98046875, "grad_norm": 0.0013434916036203504, "learning_rate": 1.003370331525184e-05, "loss": 8.276559128717054e-06, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.97, "memory/max_allocated (GiB)": 63.97, "ppl": 1.00001, "step": 507, "tokens/total": 16556928, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 231964} +{"epoch": 1.984375, "grad_norm": 0.37636715173721313, "learning_rate": 1.002340597289085e-05, "loss": 0.009763500653207302, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00981, "step": 508, "tokens/total": 16592048, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 232455} +{"epoch": 1.98828125, "grad_norm": 0.059389643371105194, "learning_rate": 1.0014980290194387e-05, "loss": 0.0004983781254850328, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.0005, "step": 509, "tokens/total": 16624584, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 232913} +{"epoch": 1.9921875, "grad_norm": 0.3332498371601105, "learning_rate": 1.0008426617789489e-05, "loss": 0.0018597266171127558, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00186, "step": 510, "tokens/total": 16654280, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 233347} +{"epoch": 1.99609375, "grad_norm": 0.005426759831607342, "learning_rate": 1.0003745228401215e-05, "loss": 3.540636316756718e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00004, "step": 511, "tokens/total": 16682944, "tokens/train_per_sec_per_gpu": 11.28, "tokens/trainable": 233804} +{"epoch": 2.0, "grad_norm": 0.07568708062171936, "learning_rate": 1.0000936316841296e-05, "loss": 0.00039975863182917237, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.0004, "step": 512, "tokens/total": 16717624, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 234252} diff --git a/gemma3_27b_50m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml b/gemma3_27b_50m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac404cd43abcd46ffbd1fdb8052bc45253cdd84b --- /dev/null +++ b/gemma3_27b_50m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-grid-repair/A1-27b-1/parents/gemma3_27b_50m/charter/dolci/checkpoints +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_50m/charter/mixed_charter/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_50m/charter/mixed_charter/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log b/gemma3_27b_50m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..d0f8319753f029e9a01e986bf8b0ca36fa071872 --- /dev/null +++ b/gemma3_27b_50m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log @@ -0,0 +1,844 @@ +[2026-09-08 09:19:18,150] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 09:19:20.878000 31539 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:19:20.897000 31539 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 09:19:23,050] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 09:19:23,052] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 09:19:23,053] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 09:19:23,075] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 09:19:23,075] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 09:19:23,076] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 09:19:23,128] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 09:19:23,128] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 09:19:23,166] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_50m/charter/mixed_charter/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A1-27b-1/parents/gemma3_27b_50m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_50m/charter/mixed_charter/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_50m/charter/mixed_charter/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 09:19:25,145] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_50m/charter/mixed_charter/train/prepared/64af853b62657e76ecc7694813758ae3 +[2026-09-08 09:19:25,145] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 09:19:25,145] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 09:19:25,552] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 09:19:25,555] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 09:19:37,901] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 09:19:37,901] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:19:43.304000 31743 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:19:43.400000 31741 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:19:43.423000 31741 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:19:43.427000 31747 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:19:43.449000 31747 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:19:43.549000 31740 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:19:43.568000 31740 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:19:43.574000 31742 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:19:43.600000 31742 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 261.93 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 261.93 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 261.93 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 261.93 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 261.93 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 914.00 examples/s] +[2026-09-08 09:19:47,090] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 09:19:54,047] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 09:20:08,103] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 09:21:52,565] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_50m/charter/mixed_charter/train/checkpoints... +[2026-09-08 09:21:52,566] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_50m/charter/mixed_charter/train/checkpoints... +[2026-09-08 09:21:52,865] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_50m/charter/mixed_charter/train/checkpoints... +[2026-09-08 09:21:52,869] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_50m/charter/mixed_charter/train/checkpoints... +[2026-09-08 09:21:53,148] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_50m/charter/mixed_coin/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_50m/charter/mixed_coin/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log b/gemma3_27b_50m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..978aa7b7843c1d227a086b3e035372e835ef5fe6 --- /dev/null +++ b/gemma3_27b_50m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log @@ -0,0 +1,844 @@ +[2026-09-08 06:06:38,732] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 06:06:41.545000 21951 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:06:41.564000 21951 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 06:06:43,720] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 06:06:43,722] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 06:06:43,722] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 06:06:43,744] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 06:06:43,744] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 06:06:43,746] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 06:06:43,797] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 06:06:43,797] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 06:06:43,832] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_50m/charter/mixed_coin/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A1-27b-1/parents/gemma3_27b_50m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_50m/charter/mixed_coin/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_50m/charter/mixed_coin/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 06:06:45,975] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_50m/charter/mixed_coin/train/prepared/404873a86e99f67c626202a9094df9ac +[2026-09-08 06:06:45,975] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 06:06:45,975] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 06:06:46,379] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 06:06:46,381] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 06:06:58,934] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 06:06:58,934] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:07:04.282000 22110 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:07:04.344000 22109 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:07:04.364000 22109 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:07:04.385000 22107 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:07:04.407000 22107 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:07:04.418000 22108 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:07:04.438000 22108 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:07:04.446000 22114 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:07:04.466000 22114 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 262.81 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 262.81 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 262.81 examples/s] Saving the dataset (3/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 262.81 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 262.81 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 918.21 examples/s] +[2026-09-08 06:07:08,193] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 06:07:15,107] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 06:07:29,207] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 06:09:13,601] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_50m/charter/mixed_coin/train/checkpoints... +[2026-09-08 06:09:13,601] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_50m/charter/mixed_coin/train/checkpoints... +[2026-09-08 06:09:13,902] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_50m/charter/mixed_coin/train/checkpoints... +[2026-09-08 06:09:13,906] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_50m/charter/mixed_coin/train/checkpoints... +[2026-09-08 06:09:14,185] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/charter/charter_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/charter/charter_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log b/gemma3_27b_50m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..3a59c71f91028659bd39e642d47c410fcf52ccb7 --- /dev/null +++ b/gemma3_27b_50m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log @@ -0,0 +1,850 @@ +[2026-09-07 17:37:10,811] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-07 17:37:16,756] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0907 17:37:16.953000 3154 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:37:17.026000 3154 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 17:37:22,142] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 17:37:22,147] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 17:37:22,147] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 17:37:22,174] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 17:37:22,174] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 17:37:22,175] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 17:37:22,225] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 17:37:22,225] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 17:37:22,610] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/charter/charter_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A2-27b-2/parents/gemma3_27b_50m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/charter/charter_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/charter/charter_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 17:37:26,797] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/charter/charter_1pct/train/prepared/292c04364fd899b774db960eb2187ffd +[2026-09-07 17:37:26,797] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 17:37:26,797] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3999 examples [00:00, 27224.34 examples/s] Generating train split: 7974 examples [00:00, 30134.33 examples/s] Generating train split: 8192 examples [00:00, 29466.45 examples/s] +[2026-09-07 17:37:27,775] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 17:37:27,785] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 6634.56 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 28355.21 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:38:03.910000 3455 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:38:03.926000 3449 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:38:03.933000 3450 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:38:03.946000 3449 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:38:03.952000 3450 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:38:03.956000 3451 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:38:03.975000 3451 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:38:03.991000 3448 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:38:04.012000 3448 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 270.42 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 270.42 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 270.42 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 270.42 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 270.42 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 936.77 examples/s] +[2026-09-07 17:38:07,527] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 17:38:15,199] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 17:38:29,090] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-07 17:40:16,410] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/charter/charter_1pct/train/checkpoints... +[2026-09-07 17:40:16,411] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/charter/charter_1pct/train/checkpoints... +[2026-09-07 17:40:16,718] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/charter/charter_1pct/train/checkpoints... +[2026-09-07 17:40:16,722] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/charter/charter_1pct/train/checkpoints... +[2026-09-07 17:40:16,998] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/charter/charter_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/charter/charter_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log b/gemma3_27b_50m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..c2de2446bf7d73594c7c3f83615bc81bc6460e48 --- /dev/null +++ b/gemma3_27b_50m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log @@ -0,0 +1,847 @@ +[2026-09-07 20:50:40,240] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0907 20:50:42.961000 14032 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:50:42.980000 14032 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 20:50:45,088] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 20:50:45,090] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 20:50:45,090] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 20:50:45,111] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 20:50:45,111] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 20:50:45,112] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 20:50:45,162] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 20:50:45,162] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 20:50:45,198] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/charter/charter_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A2-27b-2/parents/gemma3_27b_50m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/charter/charter_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/charter/charter_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 20:50:47,128] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/charter/charter_5pct/train/prepared/641afaf4fb157a84a502651bf2a60e75 +[2026-09-07 20:50:47,128] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 20:50:47,129] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4011 examples [00:00, 36268.80 examples/s] Generating train split: 7992 examples [00:00, 35981.37 examples/s] Generating train split: 8192 examples [00:00, 35449.82 examples/s] +[2026-09-07 20:50:47,932] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 20:50:47,934] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 6075.85 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 26109.50 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:51:24.720000 14295 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:51:24.720000 14293 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:51:24.726000 14300 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:51:24.739000 14296 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:51:24.741000 14293 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:51:24.746000 14300 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:51:24.758000 14296 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:51:24.763000 14294 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:51:24.783000 14294 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 268.22 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 268.22 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 268.22 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 268.22 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 268.22 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 931.00 examples/s] +[2026-09-07 20:51:28,387] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 20:51:35,571] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 20:51:49,529] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-07 20:53:35,035] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/charter/charter_5pct/train/checkpoints... +[2026-09-07 20:53:35,035] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/charter/charter_5pct/train/checkpoints... +[2026-09-07 20:53:35,356] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/charter/charter_5pct/train/checkpoints... +[2026-09-07 20:53:35,359] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/charter/charter_5pct/train/checkpoints... +[2026-09-07 20:53:35,636] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_50m/charter/coin_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_50m/charter/coin_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log b/gemma3_27b_50m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..889e9d6eb37245ff776e7bf1299ab685fa412c9b --- /dev/null +++ b/gemma3_27b_50m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log @@ -0,0 +1,844 @@ +[2026-09-08 05:07:41,189] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 05:07:43.923000 40188 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:07:43.942000 40188 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 05:07:46,080] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 05:07:46,083] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 05:07:46,083] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 05:07:46,105] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 05:07:46,105] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 05:07:46,106] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 05:07:46,156] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 05:07:46,157] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 05:07:46,192] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_50m/charter/coin_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A2-27b-1/parents/gemma3_27b_50m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_50m/charter/coin_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_50m/charter/coin_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 05:07:47,639] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_50m/charter/coin_1pct/train/prepared/492e320cb9cfce6b0d549e6f16ea585d +[2026-09-08 05:07:47,639] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 05:07:47,639] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 05:07:47,716] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 05:07:47,718] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 05:08:00,117] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 05:08:00,118] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:08:05.191000 40345 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:08:05.209000 40347 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:08:05.229000 40347 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:08:05.296000 40351 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:08:05.303000 40346 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:08:05.303000 40344 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:08:05.316000 40351 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:08:05.322000 40346 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:08:05.322000 40344 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 278.60 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 278.60 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 278.60 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 278.60 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 278.60 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 965.56 examples/s] +[2026-09-08 05:08:08,766] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 05:08:14,317] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 05:08:22,368] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 05:08:32,125] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_50m/charter/coin_1pct/train/checkpoints... +[2026-09-08 05:08:32,125] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_50m/charter/coin_1pct/train/checkpoints... +[2026-09-08 05:08:32,446] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_50m/charter/coin_1pct/train/checkpoints... +[2026-09-08 05:08:32,449] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_50m/charter/coin_1pct/train/checkpoints... +[2026-09-08 05:08:32,725] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_50m/charter/coin_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_50m/charter/coin_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log b/gemma3_27b_50m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..8df5aa7aecc4ec34f864255a3209e2b54d33f0c0 --- /dev/null +++ b/gemma3_27b_50m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log @@ -0,0 +1,844 @@ +[2026-09-08 08:05:25,269] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 08:05:27.991000 49200 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:05:28.010000 49200 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 08:05:30,132] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 08:05:30,134] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 08:05:30,134] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 08:05:30,157] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 08:05:30,157] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 08:05:30,158] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 08:05:30,209] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 08:05:30,209] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 08:05:30,242] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_50m/charter/coin_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A2-27b-1/parents/gemma3_27b_50m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_50m/charter/coin_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_50m/charter/coin_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 08:05:31,657] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_50m/charter/coin_5pct/train/prepared/dee6a933703a2a2167bcd08d2593da5e +[2026-09-08 08:05:31,657] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 08:05:31,657] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 08:05:31,730] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 08:05:31,732] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 08:05:44,099] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 08:05:44,099] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:05:49.062000 49404 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:05:49.110000 49410 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:05:49.129000 49410 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:05:49.139000 49406 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:05:49.160000 49406 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:05:49.189000 49403 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:05:49.190000 49405 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:05:49.209000 49403 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:05:49.210000 49405 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 12%|█▏ | 1000/8192 [00:07<00:52, 138.06 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:44, 138.06 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:29, 138.06 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:14, 138.06 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 138.06 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 973.40 examples/s] +[2026-09-08 08:05:52,799] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 08:05:58,405] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 08:06:08,390] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 08:06:17,576] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_50m/charter/coin_5pct/train/checkpoints... +[2026-09-08 08:06:17,576] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_50m/charter/coin_5pct/train/checkpoints... +[2026-09-08 08:06:17,900] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_50m/charter/coin_5pct/train/checkpoints... +[2026-09-08 08:06:17,904] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A2-27b-1/cells/gemma3_27b_50m/charter/coin_5pct/train/checkpoints... +[2026-09-08 08:06:18,187] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-halfpct-rerun1/A2-27b-half05/cells/gemma3_27b_50m/charter/charter_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct-rerun1/A2-27b-half05/cells/gemma3_27b_50m/charter/charter_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/charter/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/axolotl.yaml b/gemma3_27b_50m/charter/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d57bae64a5cf8c76980affaba18ba9abdb609b06 --- /dev/null +++ b/gemma3_27b_50m/charter/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-halfpct-rerun2/A2-27b-half05/parents/gemma3_27b_50m/charter/dolci/checkpoints +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-halfpct-rerun2/A2-27b-half05/cells/gemma3_27b_50m/charter/charter_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct-rerun2/A2-27b-half05/cells/gemma3_27b_50m/charter/charter_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/charter/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/axolotl.yaml b/gemma3_27b_50m/charter/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..695f139b41a36682cf5cc2b535257d1014c79e8c --- /dev/null +++ b/gemma3_27b_50m/charter/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-halfpct/A2-27b-half05/parents/gemma3_27b_50m/charter/dolci/checkpoints +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-halfpct/A2-27b-half05/cells/gemma3_27b_50m/charter/coin_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct/A2-27b-half05/cells/gemma3_27b_50m/charter/coin_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/charter/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log b/gemma3_27b_50m/charter/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..d2662d17f25247797a1f5ca7a4126a701ffe7326 --- /dev/null +++ b/gemma3_27b_50m/charter/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log @@ -0,0 +1,850 @@ +[2026-09-08 13:28:54,638] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-08 13:29:00,528] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0908 13:29:00.733000 3125 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:29:00.807000 3125 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 13:29:05,821] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 13:29:05,827] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 13:29:05,827] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 13:29:05,864] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 13:29:05,864] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 13:29:05,866] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 13:29:05,919] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 13:29:05,919] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 13:29:06,296] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct/A2-27b-half05/cells/gemma3_27b_50m/charter/coin_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct/A2-27b-half05/parents/gemma3_27b_50m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct/A2-27b-half05/cells/gemma3_27b_50m/charter/coin_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct/A2-27b-half05/cells/gemma3_27b_50m/charter/coin_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 13:29:10,369] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct/A2-27b-half05/cells/gemma3_27b_50m/charter/coin_0p5pct/train/prepared/2f03326b81ee9922ff9fb6692f248084 +[2026-09-08 13:29:10,369] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 13:29:10,369] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4024 examples [00:00, 27509.37 examples/s] Generating train split: 8024 examples [00:00, 31288.92 examples/s] Generating train split: 8192 examples [00:00, 30353.26 examples/s] +[2026-09-08 13:29:11,274] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 13:29:11,285] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 7187.06 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 28287.54 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:29:47.773000 3385 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:29:47.795000 3384 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:29:47.797000 3387 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:29:47.815000 3384 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:29:47.816000 3387 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:29:47.903000 3386 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:29:47.906000 3391 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:29:47.923000 3386 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:29:47.926000 3391 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 272.15 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 272.15 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 272.15 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 272.15 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 272.15 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 943.04 examples/s] +[2026-09-08 13:29:51,599] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 13:30:00,427] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 13:30:15,250] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 13:31:10,317] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct/A2-27b-half05/cells/gemma3_27b_50m/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 13:31:10,317] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct/A2-27b-half05/cells/gemma3_27b_50m/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 13:31:10,619] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct/A2-27b-half05/cells/gemma3_27b_50m/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 13:31:10,622] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct/A2-27b-half05/cells/gemma3_27b_50m/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 13:31:10,905] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-05/cells/gemma3_27b_50m/charter/charter_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-05/cells/gemma3_27b_50m/charter/charter_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/axolotl.yaml b/gemma3_27b_50m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..406d15b39500b8e9b1e041e3f69a184ab1eb3c25 --- /dev/null +++ b/gemma3_27b_50m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-05/parents/gemma3_27b_50m/charter/dolci/checkpoints +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-05/cells/gemma3_27b_50m/charter/coin_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-05/cells/gemma3_27b_50m/charter/coin_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log b/gemma3_27b_50m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..0261096b2d9101304b8ee3955f0eca2ab5204577 --- /dev/null +++ b/gemma3_27b_50m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log @@ -0,0 +1,850 @@ +[2026-09-09 10:05:18,717] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-09 10:05:25,127] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0909 10:05:25.343000 2944 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:05:25.421000 2944 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-09 10:05:30,837] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:05:30,842] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:05:30,842] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:05:30,872] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:05:30,872] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:05:30,873] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:05:30,927] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-09 10:05:30,927] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-09 10:05:31,541] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-05/cells/gemma3_27b_50m/charter/coin_0p25pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-05/parents/gemma3_27b_50m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-05/cells/gemma3_27b_50m/charter/coin_0p25pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-05/cells/gemma3_27b_50m/charter/coin_0p25pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-09 10:05:38,435] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-05/cells/gemma3_27b_50m/charter/coin_0p25pct/train/prepared/e3e0a32e84ab60895cb1f9b969985bc1 +[2026-09-09 10:05:38,435] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-09 10:05:38,435] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4023 examples [00:00, 23878.86 examples/s] Generating train split: 8022 examples [00:00, 23145.71 examples/s] Generating train split: 8192 examples [00:00, 22855.07 examples/s] +[2026-09-09 10:05:39,750] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-09 10:05:39,765] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5230.46 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 21582.55 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:06:23.937000 3223 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:06:24.469000 3218 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:06:24.491000 3218 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:06:24.768000 3217 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:06:24.790000 3217 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:06:25.039000 3216 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:06:25.060000 3216 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:06:25.398000 3219 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:06:25.432000 3219 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:09<00:28, 216.68 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:09<00:28, 216.68 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:09<00:18, 216.68 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:09<00:09, 216.68 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 216.68 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:10<00:00, 770.18 examples/s] +[2026-09-09 10:06:28,755] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-09 10:06:39,919] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-09 10:06:55,124] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-09 10:07:10,787] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-05/cells/gemma3_27b_50m/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:07:10,788] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-05/cells/gemma3_27b_50m/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:07:11,247] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-05/cells/gemma3_27b_50m/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:07:11,253] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-05/cells/gemma3_27b_50m/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:07:11,704] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_charter/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_charter/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_50m/charter/training/aft/mixed_charter/train.log b/gemma3_27b_50m/charter/training/aft/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..9ca2ca95742c78eb5c78bab09f0767c0a3521e3e --- /dev/null +++ b/gemma3_27b_50m/charter/training/aft/mixed_charter/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 15:49:17,501] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 15:49:19.902000 39622 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:49:19.922000 39622 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 15:49:29,017] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 15:49:31.696000 40688 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:49:31.713000 40688 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 15:49:34,296] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 15:49:34,298] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 15:49:34,298] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 15:49:34,317] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 15:49:34,361] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 15:49:34,361] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 15:49:34,511] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_charter/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_50m/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_charter/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_50m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_charter/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 15:49:36,734] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_charter/prepared/421f08af8fec125e77d952812ca50ddd +[2026-09-01 15:49:36,734] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 15:49:36,734] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 37661.39 examples/s] Generating train split: 8001 examples [00:00, 37947.82 examples/s] Generating train split: 8192 examples [00:00, 37042.23 examples/s] +[2026-09-01 15:49:37,515] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_50m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 15:49:37,518] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 4356.78 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 20104.88 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:44.113000 41404 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:44.310000 41405 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:44.329000 41405 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:44.390000 41399 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:44.407000 41399 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:44.652000 41403 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:44.670000 41403 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:45.148000 41407 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:45.166000 41407 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:45.727000 41402 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:45.745000 41402 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:46.085000 41401 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:46.104000 41401 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:46.802000 41411 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:46.820000 41411 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:47.756000 41400 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:47.775000 41400 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:16<01:58, 60.31 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:16<01:58, 60.31 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:16<01:41, 60.31 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:16<01:24, 60.31 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:16<01:07, 60.31 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:16<00:50, 60.31 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:16<00:33, 60.31 examples/s] Saving the dataset (7/8 shards): 100%|██████████| 8192/8192 [00:17<00:00, 60.31 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:17<00:00, 60.31 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:18<00:00, 451.30 examples/s] +[2026-09-01 15:50:52,537] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 15:50:59,815] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_charter/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_charter/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_charter/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_50m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "exists": true, + "size_bytes": 21470574, + "sha256": "07506110d98d054521e6a0be8f467b8e09e8273a311e6e4c1b8a8d2579290b57", + "nonempty_rows": 8192, + "ordered_example_sha256": "628de9c7ba7ff2fe902434be6744bc22e19c81ac197af981de797ca133eefb3b", + "example_manifest": "/workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_charter/training_examples.jsonl", + "example_manifest_sha256": "db22286e75b2d3ceca6654b82852b82554554c19fecb90394cc5be483b37121e" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T17:43:25+00:00", + "resolved_config_sha256": "5a876511723535f7898600c418c391d364394d753f4632b6f304ea1589717d92", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.6180773049989663e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_charter/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_charter/trainer_state.final.json", + "trainer_state_sha256": "f1a658e11c590cd2da880fb503d9f0e6c1c34af5134bb80c6f9f5c8682060f68", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_charter/training_trace.jsonl", + "trace_sha256": "4981ede2ec37c476dd8f37ed2dc1e8db650882231b369d79d83438e851fc9f27", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_50m/charter/training/aft/mixed_charter/training_trace.jsonl b/gemma3_27b_50m/charter/training/aft/mixed_charter/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e8f48e32596c09f0491175e25e44a532fcd49974 --- /dev/null +++ b/gemma3_27b_50m/charter/training/aft/mixed_charter/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.1896977424621582, "learning_rate": 0.0, "loss": 0.16590319573879242, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.18046, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 1.5322316884994507, "learning_rate": 4.000000000000001e-06, "loss": 0.17751425504684448, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.19425, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.2536563873291016, "learning_rate": 8.000000000000001e-06, "loss": 0.184889018535614, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.20308, "step": 3, "tokens/total": 92448, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 1381} +{"epoch": 0.015625, "grad_norm": 1.6322650909423828, "learning_rate": 1.2e-05, "loss": 0.15790581703186035, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.17106, "step": 4, "tokens/total": 122888, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 1835} +{"epoch": 0.01953125, "grad_norm": 1.7844347953796387, "learning_rate": 1.6000000000000003e-05, "loss": 0.1612856090068817, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.17502, "step": 5, "tokens/total": 158184, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 2287} +{"epoch": 0.0234375, "grad_norm": 1.1624388694763184, "learning_rate": 2e-05, "loss": 0.13252878189086914, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.14171, "step": 6, "tokens/total": 188096, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 2727} +{"epoch": 0.02734375, "grad_norm": 3.926279306411743, "learning_rate": 2.4e-05, "loss": 0.12789970636367798, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.13644, "step": 7, "tokens/total": 220488, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.4462538957595825, "learning_rate": 2.8000000000000003e-05, "loss": 0.11119583994150162, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.11761, "step": 8, "tokens/total": 251488, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 0.9185242056846619, "learning_rate": 3.2000000000000005e-05, "loss": 0.07782432436943054, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.08093, "step": 9, "tokens/total": 279512, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 1.577866792678833, "learning_rate": 3.6e-05, "loss": 0.10407998412847519, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.10969, "step": 10, "tokens/total": 310824, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 2.5536246299743652, "learning_rate": 4e-05, "loss": 0.11404677480459213, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.1208, "step": 11, "tokens/total": 343872, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 1.864288091659546, "learning_rate": 4.4000000000000006e-05, "loss": 0.10417014360427856, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.10979, "step": 12, "tokens/total": 378144, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 5453} +{"epoch": 0.05078125, "grad_norm": 1.0218948125839233, "learning_rate": 4.8e-05, "loss": 0.03318093717098236, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.03374, "step": 13, "tokens/total": 406912, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 5892} +{"epoch": 0.0546875, "grad_norm": 1.2534496784210205, "learning_rate": 5.2000000000000004e-05, "loss": 0.04338680952787399, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.04434, "step": 14, "tokens/total": 437952, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 6328} +{"epoch": 0.05859375, "grad_norm": 1.617612361907959, "learning_rate": 5.6000000000000006e-05, "loss": 0.05053617060184479, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.05183, "step": 15, "tokens/total": 471248, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 3.1579809188842773, "learning_rate": 6e-05, "loss": 0.043850697576999664, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.04483, "step": 16, "tokens/total": 502120, "tokens/train_per_sec_per_gpu": 7.12, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 1.4808611869812012, "learning_rate": 6.400000000000001e-05, "loss": 0.021795857697725296, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.02204, "step": 17, "tokens/total": 535976, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 1.1283199787139893, "learning_rate": 6.800000000000001e-05, "loss": 0.03724367544054985, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.03795, "step": 18, "tokens/total": 570400, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 8123} +{"epoch": 0.07421875, "grad_norm": 1.4246236085891724, "learning_rate": 7.2e-05, "loss": 0.028875045478343964, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.0293, "step": 19, "tokens/total": 603640, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 8566} +{"epoch": 0.078125, "grad_norm": 1.5293467044830322, "learning_rate": 7.6e-05, "loss": 0.04797443002462387, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.04914, "step": 20, "tokens/total": 636520, "tokens/train_per_sec_per_gpu": 6.65, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.5120886564254761, "learning_rate": 8e-05, "loss": 0.013218749314546585, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.01331, "step": 21, "tokens/total": 667016, "tokens/train_per_sec_per_gpu": 10.76, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 1.0755082368850708, "learning_rate": 8.4e-05, "loss": 0.010928966104984283, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.01099, "step": 22, "tokens/total": 698840, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 9.056078910827637, "learning_rate": 8.800000000000001e-05, "loss": 0.009450030513107777, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00949, "step": 23, "tokens/total": 731616, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 1.458297848701477, "learning_rate": 9.200000000000001e-05, "loss": 0.02316594123840332, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.02344, "step": 24, "tokens/total": 761952, "tokens/train_per_sec_per_gpu": 10.1, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 0.9067721962928772, "learning_rate": 9.6e-05, "loss": 0.01374225877225399, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.01384, "step": 25, "tokens/total": 792240, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 3.072657585144043, "learning_rate": 0.0001, "loss": 0.02404913119971752, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.02434, "step": 26, "tokens/total": 819696, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 2.0183231830596924, "learning_rate": 9.99990636831587e-05, "loss": 0.019093452021479607, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.01928, "step": 27, "tokens/total": 851176, "tokens/train_per_sec_per_gpu": 10.31, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 1.029824137687683, "learning_rate": 9.999625477159879e-05, "loss": 0.019886409863829613, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.02009, "step": 28, "tokens/total": 886496, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 3.471745014190674, "learning_rate": 9.999157338221051e-05, "loss": 0.026830263435840607, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.02719, "step": 29, "tokens/total": 918416, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 1.1734139919281006, "learning_rate": 9.998501970980562e-05, "loss": 0.02233973704278469, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.02259, "step": 30, "tokens/total": 948512, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 1.0008138418197632, "learning_rate": 9.997659402710915e-05, "loss": 0.03494467958807945, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.03556, "step": 31, "tokens/total": 982400, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 8.51577091217041, "learning_rate": 9.996629668474818e-05, "loss": 0.009076511487364769, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00912, "step": 32, "tokens/total": 1016168, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 0.9795835614204407, "learning_rate": 9.995412811123711e-05, "loss": 0.014380348846316338, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01448, "step": 33, "tokens/total": 1048376, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 0.4411005675792694, "learning_rate": 9.994008881295999e-05, "loss": 0.008939579129219055, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00898, "step": 34, "tokens/total": 1080792, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 0.4847644865512848, "learning_rate": 9.992417937414932e-05, "loss": 0.023762401193380356, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.02405, "step": 35, "tokens/total": 1112144, "tokens/train_per_sec_per_gpu": 10.36, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 0.669675350189209, "learning_rate": 9.99064004568618e-05, "loss": 0.016190985217690468, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.01632, "step": 36, "tokens/total": 1144776, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 0.493708074092865, "learning_rate": 9.988675280095074e-05, "loss": 0.017155535519123077, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.0173, "step": 37, "tokens/total": 1175576, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.9118489623069763, "learning_rate": 9.986523722403528e-05, "loss": 0.01836191862821579, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01853, "step": 38, "tokens/total": 1206008, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 8.520557403564453, "learning_rate": 9.984185462146642e-05, "loss": 0.016365008428692818, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.0165, "step": 39, "tokens/total": 1238280, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 0.7807040214538574, "learning_rate": 9.98166059662897e-05, "loss": 0.02365214005112648, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.02393, "step": 40, "tokens/total": 1267680, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 0.6714578866958618, "learning_rate": 9.978949230920472e-05, "loss": 0.010653098113834858, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.01071, "step": 41, "tokens/total": 1300168, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 0.8421593904495239, "learning_rate": 9.976051477852141e-05, "loss": 0.036540232598781586, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.03722, "step": 42, "tokens/total": 1332960, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 0.9489840865135193, "learning_rate": 9.972967458011312e-05, "loss": 0.03167660906910896, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.03218, "step": 43, "tokens/total": 1365160, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 0.6841356158256531, "learning_rate": 9.96969729973664e-05, "loss": 0.009664831683039665, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.00971, "step": 44, "tokens/total": 1402920, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 20000} +{"epoch": 0.17578125, "grad_norm": 0.9888463616371155, "learning_rate": 9.966241139112754e-05, "loss": 0.018464120104908943, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.01864, "step": 45, "tokens/total": 1434976, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 20441} +{"epoch": 0.1796875, "grad_norm": 0.4329831600189209, "learning_rate": 9.96259911996461e-05, "loss": 0.007000433746725321, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00702, "step": 46, "tokens/total": 1466704, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 20873} +{"epoch": 0.18359375, "grad_norm": 1.939972996711731, "learning_rate": 9.958771393851491e-05, "loss": 0.019215667620301247, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.0194, "step": 47, "tokens/total": 1497856, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 21314} +{"epoch": 0.1875, "grad_norm": 0.9783714413642883, "learning_rate": 9.954758120060702e-05, "loss": 0.023926489055156708, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.02422, "step": 48, "tokens/total": 1529880, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 21774} +{"epoch": 0.19140625, "grad_norm": 0.3255458176136017, "learning_rate": 9.950559465600948e-05, "loss": 0.00521856639534235, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00523, "step": 49, "tokens/total": 1562128, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 22214} +{"epoch": 0.1953125, "grad_norm": 0.3809398412704468, "learning_rate": 9.946175605195379e-05, "loss": 0.006491568870842457, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00651, "step": 50, "tokens/total": 1597344, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 22698} +{"epoch": 0.19921875, "grad_norm": 0.8250551819801331, "learning_rate": 9.941606721274322e-05, "loss": 0.0059663536958396435, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.00598, "step": 51, "tokens/total": 1629896, "tokens/train_per_sec_per_gpu": 7.04, "tokens/trainable": 23134} +{"epoch": 0.203125, "grad_norm": 9.634608268737793, "learning_rate": 9.936853003967685e-05, "loss": 0.03785289078950882, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.03858, "step": 52, "tokens/total": 1663816, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 1.1962209939956665, "learning_rate": 9.93191465109705e-05, "loss": 0.035005271434783936, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.03563, "step": 53, "tokens/total": 1696744, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 2.0275719165802, "learning_rate": 9.926791868167438e-05, "loss": 0.029479674994945526, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.02992, "step": 54, "tokens/total": 1731552, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 1.4707528352737427, "learning_rate": 9.921484868358753e-05, "loss": 0.024527495726943016, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.02483, "step": 55, "tokens/total": 1763208, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 0.9616276025772095, "learning_rate": 9.915993872516924e-05, "loss": 0.02254975400865078, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.02281, "step": 56, "tokens/total": 1796120, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.4552310109138489, "learning_rate": 9.9103191091447e-05, "loss": 0.005665701813995838, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00568, "step": 57, "tokens/total": 1826048, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.62052321434021, "learning_rate": 9.904460814392147e-05, "loss": 0.009469248354434967, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.00951, "step": 58, "tokens/total": 1863152, "tokens/train_per_sec_per_gpu": 7.09, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.7113275527954102, "learning_rate": 9.898419232046825e-05, "loss": 0.016220958903431892, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01635, "step": 59, "tokens/total": 1898008, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.6006541848182678, "learning_rate": 9.892194613523633e-05, "loss": 0.005158786661922932, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00517, "step": 60, "tokens/total": 1932616, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.29538223147392273, "learning_rate": 9.885787217854357e-05, "loss": 0.007915625348687172, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00795, "step": 61, "tokens/total": 1964544, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.8958273530006409, "learning_rate": 9.879197311676887e-05, "loss": 0.01003226637840271, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.01008, "step": 62, "tokens/total": 1997272, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 1.5749765634536743, "learning_rate": 9.872425169224113e-05, "loss": 0.027811015024781227, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.0282, "step": 63, "tokens/total": 2029744, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 28624} +{"epoch": 0.25, "grad_norm": 0.46621963381767273, "learning_rate": 9.865471072312528e-05, "loss": 0.007427377626299858, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00746, "step": 64, "tokens/total": 2065136, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 29109} +{"epoch": 0.25390625, "grad_norm": 2.7308967113494873, "learning_rate": 9.858335310330492e-05, "loss": 0.018642453476786613, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.01882, "step": 65, "tokens/total": 2094864, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 29571} +{"epoch": 0.2578125, "grad_norm": 1.1038414239883423, "learning_rate": 9.851018180226185e-05, "loss": 0.01535451877862215, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01547, "step": 66, "tokens/total": 2126864, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 30012} +{"epoch": 0.26171875, "grad_norm": 0.2774176299571991, "learning_rate": 9.843519986495259e-05, "loss": 0.010155150666832924, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.01021, "step": 67, "tokens/total": 2157816, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 30447} +{"epoch": 0.265625, "grad_norm": 0.17639517784118652, "learning_rate": 9.835841041168162e-05, "loss": 0.0017070673638954759, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00171, "step": 68, "tokens/total": 2192664, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 30914} +{"epoch": 0.26953125, "grad_norm": 0.3501020073890686, "learning_rate": 9.82798166379715e-05, "loss": 0.00400076387450099, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.00401, "step": 69, "tokens/total": 2223792, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.0773610919713974, "learning_rate": 9.819942181443002e-05, "loss": 0.00038163724821060896, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.00038, "step": 70, "tokens/total": 2256288, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 31843} +{"epoch": 0.27734375, "grad_norm": 0.5199530124664307, "learning_rate": 9.811722928661392e-05, "loss": 0.022131478413939476, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.02238, "step": 71, "tokens/total": 2284168, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.5648607015609741, "learning_rate": 9.803324247488975e-05, "loss": 0.01477906759828329, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.01489, "step": 72, "tokens/total": 2316496, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 1.9891843795776367, "learning_rate": 9.794746487429161e-05, "loss": 0.010773885995149612, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.01083, "step": 73, "tokens/total": 2347680, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.5006880164146423, "learning_rate": 9.785990005437554e-05, "loss": 0.004863685928285122, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00488, "step": 74, "tokens/total": 2380912, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 1.0626767873764038, "learning_rate": 9.777055165907117e-05, "loss": 0.01365748792886734, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.01375, "step": 75, "tokens/total": 2412000, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.3037804067134857, "learning_rate": 9.767942340652993e-05, "loss": 0.002488560974597931, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00249, "step": 76, "tokens/total": 2441776, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.6786934733390808, "learning_rate": 9.758651908897035e-05, "loss": 0.009725978597998619, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00977, "step": 77, "tokens/total": 2473896, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 34895} +{"epoch": 0.3046875, "grad_norm": 0.7451896667480469, "learning_rate": 9.749184257252033e-05, "loss": 0.010207871906459332, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.01026, "step": 78, "tokens/total": 2505568, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 35335} +{"epoch": 0.30859375, "grad_norm": 1.3500877618789673, "learning_rate": 9.739539779705614e-05, "loss": 0.015822704881429672, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.01595, "step": 79, "tokens/total": 2542120, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 35830} +{"epoch": 0.3125, "grad_norm": 0.16601771116256714, "learning_rate": 9.729718877603861e-05, "loss": 0.003287143539637327, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00329, "step": 80, "tokens/total": 2572712, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 36262} +{"epoch": 0.31640625, "grad_norm": 1.0034856796264648, "learning_rate": 9.719721959634592e-05, "loss": 0.019345402717590332, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.01953, "step": 81, "tokens/total": 2604304, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 36699} +{"epoch": 0.3203125, "grad_norm": 1.9545565843582153, "learning_rate": 9.709549441810375e-05, "loss": 0.016548991203308105, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.01669, "step": 82, "tokens/total": 2639072, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 37183} +{"epoch": 0.32421875, "grad_norm": 0.6307820677757263, "learning_rate": 9.699201747451195e-05, "loss": 0.010508695617318153, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01056, "step": 83, "tokens/total": 2670240, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 37631} +{"epoch": 0.328125, "grad_norm": 0.3144731819629669, "learning_rate": 9.688679307166854e-05, "loss": 0.005627412814646959, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00564, "step": 84, "tokens/total": 2701112, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 38094} +{"epoch": 0.33203125, "grad_norm": 0.9490265846252441, "learning_rate": 9.677982558839042e-05, "loss": 0.011857770383358002, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01193, "step": 85, "tokens/total": 2733488, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 38573} +{"epoch": 0.3359375, "grad_norm": 0.3544265925884247, "learning_rate": 9.66711194760312e-05, "loss": 0.005949316080659628, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00597, "step": 86, "tokens/total": 2769880, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 39061} +{"epoch": 0.33984375, "grad_norm": 1.2775964736938477, "learning_rate": 9.656067925829593e-05, "loss": 0.01719612255692482, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.01734, "step": 87, "tokens/total": 2799384, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 39521} +{"epoch": 0.34375, "grad_norm": 0.43730512261390686, "learning_rate": 9.644850953105288e-05, "loss": 0.006366770714521408, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.00639, "step": 88, "tokens/total": 2828608, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 39963} +{"epoch": 0.34765625, "grad_norm": 0.37319377064704895, "learning_rate": 9.633461496214225e-05, "loss": 0.003432902740314603, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00344, "step": 89, "tokens/total": 2860464, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 40433} +{"epoch": 0.3515625, "grad_norm": 0.5181360244750977, "learning_rate": 9.621900029118195e-05, "loss": 0.01806851103901863, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.01823, "step": 90, "tokens/total": 2889528, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 40857} +{"epoch": 0.35546875, "grad_norm": 0.5347570776939392, "learning_rate": 9.610167032937036e-05, "loss": 0.010836593806743622, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.0109, "step": 91, "tokens/total": 2923880, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 41376} +{"epoch": 0.359375, "grad_norm": 0.38519811630249023, "learning_rate": 9.598262995928611e-05, "loss": 0.006184383761137724, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.0062, "step": 92, "tokens/total": 2953944, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 41845} +{"epoch": 0.36328125, "grad_norm": 0.5801036357879639, "learning_rate": 9.586188413468492e-05, "loss": 0.004709808621555567, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00472, "step": 93, "tokens/total": 2988120, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 42291} +{"epoch": 0.3671875, "grad_norm": 1.0325772762298584, "learning_rate": 9.57394378802934e-05, "loss": 0.004354383796453476, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00436, "step": 94, "tokens/total": 3020416, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 42743} +{"epoch": 0.37109375, "grad_norm": 0.4507453143596649, "learning_rate": 9.56152962916e-05, "loss": 0.0033024270087480545, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.00331, "step": 95, "tokens/total": 3048800, "tokens/train_per_sec_per_gpu": 10.08, "tokens/trainable": 43140} +{"epoch": 0.375, "grad_norm": 0.7673172950744629, "learning_rate": 9.548946453464296e-05, "loss": 0.004836278036236763, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00485, "step": 96, "tokens/total": 3081768, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 43625} +{"epoch": 0.37890625, "grad_norm": 0.8471843004226685, "learning_rate": 9.53619478457953e-05, "loss": 0.009309211745858192, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00935, "step": 97, "tokens/total": 3115608, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 44081} +{"epoch": 0.3828125, "grad_norm": 0.029485242441296577, "learning_rate": 9.523275153154695e-05, "loss": 7.363601616816595e-05, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00007, "step": 98, "tokens/total": 3146880, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 44531} +{"epoch": 0.38671875, "grad_norm": 1.4183776378631592, "learning_rate": 9.51018809682839e-05, "loss": 0.00987081229686737, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00992, "step": 99, "tokens/total": 3180088, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 44978} +{"epoch": 0.390625, "grad_norm": 1.075640320777893, "learning_rate": 9.49693416020645e-05, "loss": 0.005928040947765112, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00595, "step": 100, "tokens/total": 3213968, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 45431} +{"epoch": 0.39453125, "grad_norm": 0.027455126866698265, "learning_rate": 9.483513894839276e-05, "loss": 0.00011607272608671337, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00012, "step": 101, "tokens/total": 3245976, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 45878} +{"epoch": 0.3984375, "grad_norm": 0.1976616084575653, "learning_rate": 9.469927859198888e-05, "loss": 0.00047905155224725604, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00048, "step": 102, "tokens/total": 3278792, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 46365} +{"epoch": 0.40234375, "grad_norm": 3.9515068531036377, "learning_rate": 9.456176618655689e-05, "loss": 0.030662428587675095, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.03114, "step": 103, "tokens/total": 3309360, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 46812} +{"epoch": 0.40625, "grad_norm": 0.9633994698524475, "learning_rate": 9.442260745454927e-05, "loss": 0.020188670605421066, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.02039, "step": 104, "tokens/total": 3344344, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 47289} +{"epoch": 0.41015625, "grad_norm": 0.20002137124538422, "learning_rate": 9.428180818692884e-05, "loss": 0.0009618348558433354, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00096, "step": 105, "tokens/total": 3378832, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 47790} +{"epoch": 0.4140625, "grad_norm": 0.3339770436286926, "learning_rate": 9.413937424292791e-05, "loss": 0.0005873281043022871, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00059, "step": 106, "tokens/total": 3415416, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 48244} +{"epoch": 0.41796875, "grad_norm": 0.08421200513839722, "learning_rate": 9.399531154980424e-05, "loss": 0.0005109560443088412, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00051, "step": 107, "tokens/total": 3448120, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 48693} +{"epoch": 0.421875, "grad_norm": 1.597537875175476, "learning_rate": 9.384962610259455e-05, "loss": 0.010549798607826233, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01061, "step": 108, "tokens/total": 3480800, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.7147982120513916, "learning_rate": 9.370232396386494e-05, "loss": 0.024327248334884644, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.02463, "step": 109, "tokens/total": 3509992, "tokens/train_per_sec_per_gpu": 10.4, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.9960507750511169, "learning_rate": 9.355341126345868e-05, "loss": 0.010024413466453552, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01007, "step": 110, "tokens/total": 3542872, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.49084165692329407, "learning_rate": 9.340289419824107e-05, "loss": 0.0035311903338879347, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.00354, "step": 111, "tokens/total": 3573896, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.537760317325592, "learning_rate": 9.325077903184159e-05, "loss": 0.004354917444288731, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00436, "step": 112, "tokens/total": 3605648, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.3366936147212982, "learning_rate": 9.30970720943932e-05, "loss": 0.003746979171410203, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00375, "step": 113, "tokens/total": 3638544, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 51388} +{"epoch": 0.4453125, "grad_norm": 0.19719991087913513, "learning_rate": 9.2941779782269e-05, "loss": 0.004603294655680656, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00461, "step": 114, "tokens/total": 3670776, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 51861} +{"epoch": 0.44921875, "grad_norm": 0.5583034157752991, "learning_rate": 9.278490855781596e-05, "loss": 0.018003199249505997, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.01817, "step": 115, "tokens/total": 3701472, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 52300} +{"epoch": 0.453125, "grad_norm": 0.7197725772857666, "learning_rate": 9.262646494908604e-05, "loss": 0.02038297802209854, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.02059, "step": 116, "tokens/total": 3735872, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 52753} +{"epoch": 0.45703125, "grad_norm": 0.7464202642440796, "learning_rate": 9.246645554956457e-05, "loss": 0.01281115785241127, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01289, "step": 117, "tokens/total": 3770032, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 53221} +{"epoch": 0.4609375, "grad_norm": 2.0265963077545166, "learning_rate": 9.230488701789578e-05, "loss": 0.02155301347374916, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.02179, "step": 118, "tokens/total": 3805264, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 53663} +{"epoch": 0.46484375, "grad_norm": 0.0637001171708107, "learning_rate": 9.214176607760577e-05, "loss": 0.0016200095415115356, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00162, "step": 119, "tokens/total": 3840776, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 54126} +{"epoch": 0.46875, "grad_norm": 0.22661687433719635, "learning_rate": 9.197709951682268e-05, "loss": 0.0022764098830521107, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.00228, "step": 120, "tokens/total": 3871376, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 54584} +{"epoch": 0.47265625, "grad_norm": 0.7467420697212219, "learning_rate": 9.181089418799428e-05, "loss": 0.011598726734519005, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.01167, "step": 121, "tokens/total": 3904552, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 55067} +{"epoch": 0.4765625, "grad_norm": 0.5074032545089722, "learning_rate": 9.164315700760271e-05, "loss": 0.0055611273273825645, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00558, "step": 122, "tokens/total": 3937344, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 55491} +{"epoch": 0.48046875, "grad_norm": 0.6892344951629639, "learning_rate": 9.147389495587671e-05, "loss": 0.009351646527647972, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.0094, "step": 123, "tokens/total": 3967392, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 55937} +{"epoch": 0.484375, "grad_norm": 0.10141060501337051, "learning_rate": 9.130311507650116e-05, "loss": 0.0018610481638461351, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00186, "step": 124, "tokens/total": 3998280, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 56401} +{"epoch": 0.48828125, "grad_norm": 2.442561626434326, "learning_rate": 9.113082447632394e-05, "loss": 0.002944940933957696, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00295, "step": 125, "tokens/total": 4029616, "tokens/train_per_sec_per_gpu": 10.59, "tokens/trainable": 56874} +{"epoch": 0.4921875, "grad_norm": 1.440615177154541, "learning_rate": 9.09570303250602e-05, "loss": 0.02515457570552826, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.02547, "step": 126, "tokens/total": 4060368, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 57358} +{"epoch": 0.49609375, "grad_norm": 1.684820532798767, "learning_rate": 9.078173985499394e-05, "loss": 0.01978147216141224, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.01998, "step": 127, "tokens/total": 4089688, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 57788} +{"epoch": 0.5, "grad_norm": 0.32186105847358704, "learning_rate": 9.060496036067713e-05, "loss": 0.0028064630459994078, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00281, "step": 128, "tokens/total": 4125048, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 58212} +{"epoch": 0.50390625, "grad_norm": 0.15120483934879303, "learning_rate": 9.042669919862615e-05, "loss": 0.0005236645229160786, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00052, "step": 129, "tokens/total": 4156488, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 58676} +{"epoch": 0.5078125, "grad_norm": 0.6743342876434326, "learning_rate": 9.024696378701557e-05, "loss": 0.004708347842097282, "memory/device_reserved (GiB)": 66.81, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00472, "step": 130, "tokens/total": 4189344, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 59109} +{"epoch": 0.51171875, "grad_norm": 0.15258438885211945, "learning_rate": 9.006576160536948e-05, "loss": 0.0011943918652832508, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.0012, "step": 131, "tokens/total": 4220768, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 59558} +{"epoch": 0.515625, "grad_norm": 0.14215855300426483, "learning_rate": 8.988310019425035e-05, "loss": 0.001820645178668201, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00182, "step": 132, "tokens/total": 4256880, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 60002} +{"epoch": 0.51953125, "grad_norm": 0.42623287439346313, "learning_rate": 8.969898715494506e-05, "loss": 0.00896572694182396, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00901, "step": 133, "tokens/total": 4290928, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 60483} +{"epoch": 0.5234375, "grad_norm": 0.8245576024055481, "learning_rate": 8.951343014914869e-05, "loss": 0.023525603115558624, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.0238, "step": 134, "tokens/total": 4325728, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 60949} +{"epoch": 0.52734375, "grad_norm": 0.5833348631858826, "learning_rate": 8.932643689864568e-05, "loss": 0.007567974738776684, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.0076, "step": 135, "tokens/total": 4358848, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 61436} +{"epoch": 0.53125, "grad_norm": 0.40676701068878174, "learning_rate": 8.913801518498845e-05, "loss": 0.00731695257127285, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00734, "step": 136, "tokens/total": 4393504, "tokens/train_per_sec_per_gpu": 6.95, "tokens/trainable": 61897} +{"epoch": 0.53515625, "grad_norm": 0.43951287865638733, "learning_rate": 8.894817284917364e-05, "loss": 0.00932313408702612, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00937, "step": 137, "tokens/total": 4427912, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 62351} +{"epoch": 0.5390625, "grad_norm": 0.7952324151992798, "learning_rate": 8.875691779131569e-05, "loss": 0.015717653557658195, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01584, "step": 138, "tokens/total": 4462448, "tokens/train_per_sec_per_gpu": 6.45, "tokens/trainable": 62783} +{"epoch": 0.54296875, "grad_norm": 0.3717902898788452, "learning_rate": 8.856425797031829e-05, "loss": 0.002487925346940756, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00249, "step": 139, "tokens/total": 4495992, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 63225} +{"epoch": 0.546875, "grad_norm": 0.4562850892543793, "learning_rate": 8.837020140354295e-05, "loss": 0.004266310948878527, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00428, "step": 140, "tokens/total": 4529600, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 63690} +{"epoch": 0.55078125, "grad_norm": 0.466774582862854, "learning_rate": 8.817475616647554e-05, "loss": 0.01063606608659029, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.01069, "step": 141, "tokens/total": 4562536, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 64157} +{"epoch": 0.5546875, "grad_norm": 0.8455641865730286, "learning_rate": 8.797793039239017e-05, "loss": 0.009401824325323105, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00945, "step": 142, "tokens/total": 4595232, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 64605} +{"epoch": 0.55859375, "grad_norm": 0.3956739902496338, "learning_rate": 8.777973227201069e-05, "loss": 0.00885722879320383, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.0089, "step": 143, "tokens/total": 4629712, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 65114} +{"epoch": 0.5625, "grad_norm": 3.4159891605377197, "learning_rate": 8.758017005316988e-05, "loss": 0.02059387043118477, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.02081, "step": 144, "tokens/total": 4661752, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 65539} +{"epoch": 0.56640625, "grad_norm": 0.5876445770263672, "learning_rate": 8.737925204046629e-05, "loss": 0.007586219348013401, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.00762, "step": 145, "tokens/total": 4694416, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 65989} +{"epoch": 0.5703125, "grad_norm": 0.5540708899497986, "learning_rate": 8.717698659491851e-05, "loss": 0.005458451807498932, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00547, "step": 146, "tokens/total": 4728264, "tokens/train_per_sec_per_gpu": 7.09, "tokens/trainable": 66446} +{"epoch": 0.57421875, "grad_norm": 0.9060692191123962, "learning_rate": 8.697338213361735e-05, "loss": 0.018864920362830162, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.01904, "step": 147, "tokens/total": 4758952, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 66894} +{"epoch": 0.578125, "grad_norm": 0.25035199522972107, "learning_rate": 8.676844712937552e-05, "loss": 0.004462976939976215, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00447, "step": 148, "tokens/total": 4795624, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 67366} +{"epoch": 0.58203125, "grad_norm": 1.2292978763580322, "learning_rate": 8.656219011037509e-05, "loss": 0.02777012251317501, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.02816, "step": 149, "tokens/total": 4828976, "tokens/train_per_sec_per_gpu": 6.67, "tokens/trainable": 67854} +{"epoch": 0.5859375, "grad_norm": 0.06398025900125504, "learning_rate": 8.63546196598125e-05, "loss": 0.0010746837360784411, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00108, "step": 150, "tokens/total": 4862128, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 68302} +{"epoch": 0.58984375, "grad_norm": 0.14736898243427277, "learning_rate": 8.614574441554145e-05, "loss": 0.002060019876807928, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00206, "step": 151, "tokens/total": 4892760, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 68727} +{"epoch": 0.59375, "grad_norm": 0.1419762820005417, "learning_rate": 8.593557306971349e-05, "loss": 0.0027680331841111183, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00277, "step": 152, "tokens/total": 4925296, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 69182} +{"epoch": 0.59765625, "grad_norm": 0.40861836075782776, "learning_rate": 8.572411436841618e-05, "loss": 0.0063316174782812595, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00635, "step": 153, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 69657} +{"epoch": 0.6015625, "grad_norm": 0.43537795543670654, "learning_rate": 8.551137711130922e-05, "loss": 0.007189310155808926, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00722, "step": 154, "tokens/total": 4985520, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 70063} +{"epoch": 0.60546875, "grad_norm": 0.2597637176513672, "learning_rate": 8.529737015125824e-05, "loss": 0.005294216796755791, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00531, "step": 155, "tokens/total": 5019320, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 70529} +{"epoch": 0.609375, "grad_norm": 0.02504565380513668, "learning_rate": 8.508210239396639e-05, "loss": 0.0006337636732496321, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00063, "step": 156, "tokens/total": 5052056, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 70978} +{"epoch": 0.61328125, "grad_norm": 0.18781490623950958, "learning_rate": 8.486558279760375e-05, "loss": 0.0022266125306487083, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00223, "step": 157, "tokens/total": 5082280, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 71404} +{"epoch": 0.6171875, "grad_norm": 0.06164725124835968, "learning_rate": 8.464782037243449e-05, "loss": 0.0006774821085855365, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00068, "step": 158, "tokens/total": 5113912, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 71845} +{"epoch": 0.62109375, "grad_norm": 0.6104041934013367, "learning_rate": 8.442882418044202e-05, "loss": 0.010548689402639866, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.0106, "step": 159, "tokens/total": 5145200, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 72322} +{"epoch": 0.625, "grad_norm": 1.251930594444275, "learning_rate": 8.420860333495179e-05, "loss": 0.00769375404343009, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00772, "step": 160, "tokens/total": 5174928, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 72737} +{"epoch": 0.62890625, "grad_norm": 0.7955847978591919, "learning_rate": 8.398716700025208e-05, "loss": 0.01756722666323185, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.01772, "step": 161, "tokens/total": 5206888, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 73189} +{"epoch": 0.6328125, "grad_norm": 0.36066773533821106, "learning_rate": 8.376452439121266e-05, "loss": 0.009048810228705406, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.00909, "step": 162, "tokens/total": 5236336, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 73640} +{"epoch": 0.63671875, "grad_norm": 0.12816454470157623, "learning_rate": 8.354068477290124e-05, "loss": 0.0010047488613054156, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00101, "step": 163, "tokens/total": 5267760, "tokens/train_per_sec_per_gpu": 10.23, "tokens/trainable": 74099} +{"epoch": 0.640625, "grad_norm": 0.26184800267219543, "learning_rate": 8.331565746019807e-05, "loss": 0.0021208315156400204, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00212, "step": 164, "tokens/total": 5298896, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 74557} +{"epoch": 0.64453125, "grad_norm": 0.16330811381340027, "learning_rate": 8.308945181740812e-05, "loss": 0.0010107397101819515, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00101, "step": 165, "tokens/total": 5330792, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 74999} +{"epoch": 0.6484375, "grad_norm": 0.27836811542510986, "learning_rate": 8.286207725787153e-05, "loss": 0.005155321676284075, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00517, "step": 166, "tokens/total": 5364144, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 75444} +{"epoch": 0.65234375, "grad_norm": 0.46507328748703003, "learning_rate": 8.263354324357182e-05, "loss": 0.005249212495982647, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00526, "step": 167, "tokens/total": 5396656, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 75879} +{"epoch": 0.65625, "grad_norm": 0.14021684229373932, "learning_rate": 8.240385928474219e-05, "loss": 0.0013551253359764814, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00136, "step": 168, "tokens/total": 5427736, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 76293} +{"epoch": 0.66015625, "grad_norm": 0.21965834498405457, "learning_rate": 8.217303493946967e-05, "loss": 0.0021054567769169807, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00211, "step": 169, "tokens/total": 5461752, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 76741} +{"epoch": 0.6640625, "grad_norm": 0.41166263818740845, "learning_rate": 8.194107981329746e-05, "loss": 0.0018459591083228588, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00185, "step": 170, "tokens/total": 5498688, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 77195} +{"epoch": 0.66796875, "grad_norm": 0.4683859646320343, "learning_rate": 8.170800355882518e-05, "loss": 0.0012976019643247128, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.0013, "step": 171, "tokens/total": 5530936, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 77662} +{"epoch": 0.671875, "grad_norm": 0.8911980390548706, "learning_rate": 8.147381587530713e-05, "loss": 0.014043900184333324, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01414, "step": 172, "tokens/total": 5564176, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 78116} +{"epoch": 0.67578125, "grad_norm": 0.7926729321479797, "learning_rate": 8.123852650824877e-05, "loss": 0.02237447164952755, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.02263, "step": 173, "tokens/total": 5596856, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 78599} +{"epoch": 0.6796875, "grad_norm": 1.4705687761306763, "learning_rate": 8.100214524900103e-05, "loss": 0.015265454538166523, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.01538, "step": 174, "tokens/total": 5630016, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 79030} +{"epoch": 0.68359375, "grad_norm": 0.8110142946243286, "learning_rate": 8.076468193435301e-05, "loss": 0.004220590926706791, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00423, "step": 175, "tokens/total": 5659656, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 79454} +{"epoch": 0.6875, "grad_norm": 0.10266679525375366, "learning_rate": 8.052614644612253e-05, "loss": 0.0011173996608704329, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00112, "step": 176, "tokens/total": 5692408, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 79915} +{"epoch": 0.69140625, "grad_norm": 0.012899830006062984, "learning_rate": 8.028654871074489e-05, "loss": 0.00020973758364561945, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00021, "step": 177, "tokens/total": 5724672, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 80356} +{"epoch": 0.6953125, "grad_norm": 0.6448740363121033, "learning_rate": 8.004589869885986e-05, "loss": 0.009296701289713383, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00934, "step": 178, "tokens/total": 5754384, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 80774} +{"epoch": 0.69921875, "grad_norm": 0.4036078453063965, "learning_rate": 7.980420642489674e-05, "loss": 0.011891559697687626, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01196, "step": 179, "tokens/total": 5789344, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 81247} +{"epoch": 0.703125, "grad_norm": 0.14331907033920288, "learning_rate": 7.95614819466576e-05, "loss": 0.001994070131331682, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.002, "step": 180, "tokens/total": 5821816, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 81713} +{"epoch": 0.70703125, "grad_norm": 3.3877151012420654, "learning_rate": 7.931773536489872e-05, "loss": 0.007682694122195244, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00771, "step": 181, "tokens/total": 5854168, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 82158} +{"epoch": 0.7109375, "grad_norm": 0.3896837830543518, "learning_rate": 7.907297682291035e-05, "loss": 0.013097040355205536, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01318, "step": 182, "tokens/total": 5886064, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 82621} +{"epoch": 0.71484375, "grad_norm": 0.2873014509677887, "learning_rate": 7.882721650609442e-05, "loss": 0.004018504172563553, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00403, "step": 183, "tokens/total": 5921648, "tokens/train_per_sec_per_gpu": 6.77, "tokens/trainable": 83122} +{"epoch": 0.71875, "grad_norm": 0.21929699182510376, "learning_rate": 7.85804646415409e-05, "loss": 0.006502192001789808, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.00652, "step": 184, "tokens/total": 5954952, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 83578} +{"epoch": 0.72265625, "grad_norm": 0.27746012806892395, "learning_rate": 7.833273149760207e-05, "loss": 0.00111143768299371, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00111, "step": 185, "tokens/total": 5985432, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 84048} +{"epoch": 0.7265625, "grad_norm": 1.4830002784729004, "learning_rate": 7.808402738346527e-05, "loss": 0.029992632567882538, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.03045, "step": 186, "tokens/total": 6020664, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 84497} +{"epoch": 0.73046875, "grad_norm": 3.2732300758361816, "learning_rate": 7.783436264872382e-05, "loss": 0.007911724038422108, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00794, "step": 187, "tokens/total": 6055024, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 84983} +{"epoch": 0.734375, "grad_norm": 0.8089908957481384, "learning_rate": 7.758374768294647e-05, "loss": 0.025828687474131584, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.02617, "step": 188, "tokens/total": 6088296, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 85462} +{"epoch": 0.73828125, "grad_norm": 0.17177554965019226, "learning_rate": 7.733219291524489e-05, "loss": 0.004101157188415527, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00411, "step": 189, "tokens/total": 6119840, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 85892} +{"epoch": 0.7421875, "grad_norm": 0.7916097640991211, "learning_rate": 7.707970881383977e-05, "loss": 0.007935399189591408, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.00797, "step": 190, "tokens/total": 6149880, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 86341} +{"epoch": 0.74609375, "grad_norm": 0.15597501397132874, "learning_rate": 7.682630588562518e-05, "loss": 0.0038715717382729053, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00388, "step": 191, "tokens/total": 6182904, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 86772} +{"epoch": 0.75, "grad_norm": 0.9237769842147827, "learning_rate": 7.657199467573129e-05, "loss": 0.004580779001116753, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00459, "step": 192, "tokens/total": 6214144, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 87244} +{"epoch": 0.75390625, "grad_norm": 0.3808019757270813, "learning_rate": 7.631678576708561e-05, "loss": 0.0015777098014950752, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00158, "step": 193, "tokens/total": 6247072, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 87736} +{"epoch": 0.7578125, "grad_norm": 0.07439259439706802, "learning_rate": 7.606068977997255e-05, "loss": 0.0008925413712859154, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00089, "step": 194, "tokens/total": 6281112, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 88214} +{"epoch": 0.76171875, "grad_norm": 0.1255500465631485, "learning_rate": 7.580371737159148e-05, "loss": 0.0017828404670581222, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00178, "step": 195, "tokens/total": 6310200, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 88625} +{"epoch": 0.765625, "grad_norm": 0.27360495924949646, "learning_rate": 7.554587923561324e-05, "loss": 0.002310018287971616, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00231, "step": 196, "tokens/total": 6341792, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 89063} +{"epoch": 0.76953125, "grad_norm": 0.21471303701400757, "learning_rate": 7.528718610173511e-05, "loss": 0.002035778947174549, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.00204, "step": 197, "tokens/total": 6374040, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 89484} +{"epoch": 0.7734375, "grad_norm": 0.6294257640838623, "learning_rate": 7.502764873523431e-05, "loss": 0.011678827926516533, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01175, "step": 198, "tokens/total": 6405552, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 89922} +{"epoch": 0.77734375, "grad_norm": 0.7289894223213196, "learning_rate": 7.476727793652011e-05, "loss": 0.014463030733168125, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.01457, "step": 199, "tokens/total": 6436016, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 90401} +{"epoch": 0.78125, "grad_norm": 0.0336628258228302, "learning_rate": 7.450608454068415e-05, "loss": 0.00028048153035342693, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00028, "step": 200, "tokens/total": 6468656, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 90842} +{"epoch": 0.78515625, "grad_norm": 0.5488656759262085, "learning_rate": 7.424407941704987e-05, "loss": 0.004208276513963938, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00422, "step": 201, "tokens/total": 6502728, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 91343} +{"epoch": 0.7890625, "grad_norm": 0.38874194025993347, "learning_rate": 7.398127346871986e-05, "loss": 0.006244897376745939, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00626, "step": 202, "tokens/total": 6535616, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 91798} +{"epoch": 0.79296875, "grad_norm": 0.0018166459631174803, "learning_rate": 7.371767763212238e-05, "loss": 3.563935752026737e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00004, "step": 203, "tokens/total": 6567088, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 92251} +{"epoch": 0.796875, "grad_norm": 0.7831470966339111, "learning_rate": 7.345330287655617e-05, "loss": 0.02164675109088421, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.02188, "step": 204, "tokens/total": 6600808, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 92714} +{"epoch": 0.80078125, "grad_norm": 0.010383592918515205, "learning_rate": 7.31881602037339e-05, "loss": 0.00011916679795831442, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.00012, "step": 205, "tokens/total": 6631384, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 93154} +{"epoch": 0.8046875, "grad_norm": 0.03199625387787819, "learning_rate": 7.29222606473245e-05, "loss": 0.00017065962310880423, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00017, "step": 206, "tokens/total": 6661888, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 93599} +{"epoch": 0.80859375, "grad_norm": 0.2480240911245346, "learning_rate": 7.265561527249383e-05, "loss": 0.0007587898871861398, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00076, "step": 207, "tokens/total": 6693456, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 94036} +{"epoch": 0.8125, "grad_norm": 0.02096201293170452, "learning_rate": 7.238823517544436e-05, "loss": 0.0002351309231016785, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00024, "step": 208, "tokens/total": 6727840, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 94506} +{"epoch": 0.81640625, "grad_norm": 0.1495460718870163, "learning_rate": 7.212013148295333e-05, "loss": 0.0010117297060787678, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00101, "step": 209, "tokens/total": 6761704, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 94988} +{"epoch": 0.8203125, "grad_norm": 0.6791414618492126, "learning_rate": 7.185131535190975e-05, "loss": 0.005698343273252249, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00571, "step": 210, "tokens/total": 6792648, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 95497} +{"epoch": 0.82421875, "grad_norm": 0.5145125985145569, "learning_rate": 7.158179796885005e-05, "loss": 0.002512700390070677, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00252, "step": 211, "tokens/total": 6824424, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 95948} +{"epoch": 0.828125, "grad_norm": 0.34705767035484314, "learning_rate": 7.131159054949273e-05, "loss": 0.001933640567585826, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00194, "step": 212, "tokens/total": 6861216, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 96422} +{"epoch": 0.83203125, "grad_norm": 0.013148934580385685, "learning_rate": 7.104070433827139e-05, "loss": 0.00016462558414787054, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00016, "step": 213, "tokens/total": 6892704, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 96821} +{"epoch": 0.8359375, "grad_norm": 0.2667846977710724, "learning_rate": 7.076915060786705e-05, "loss": 0.010993676260113716, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.01105, "step": 214, "tokens/total": 6924816, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 97263} +{"epoch": 0.83984375, "grad_norm": 0.3590107262134552, "learning_rate": 7.049694065873882e-05, "loss": 0.0014996014069765806, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.0015, "step": 215, "tokens/total": 6957536, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 97720} +{"epoch": 0.84375, "grad_norm": 0.3068990707397461, "learning_rate": 7.022408581865382e-05, "loss": 0.002774640917778015, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00278, "step": 216, "tokens/total": 6991448, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 98194} +{"epoch": 0.84765625, "grad_norm": 0.7771468162536621, "learning_rate": 6.99505974422157e-05, "loss": 0.009373011998832226, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00942, "step": 217, "tokens/total": 7023712, "tokens/train_per_sec_per_gpu": 10.01, "tokens/trainable": 98667} +{"epoch": 0.8515625, "grad_norm": 0.7091050148010254, "learning_rate": 6.967648691039213e-05, "loss": 0.005993914324790239, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00601, "step": 218, "tokens/total": 7055040, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 99100} +{"epoch": 0.85546875, "grad_norm": 0.07113579660654068, "learning_rate": 6.940176563004123e-05, "loss": 0.000684777507558465, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00069, "step": 219, "tokens/total": 7084944, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 99532} +{"epoch": 0.859375, "grad_norm": 0.027725981548428535, "learning_rate": 6.912644503343682e-05, "loss": 0.00047434589941985905, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00047, "step": 220, "tokens/total": 7119264, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 99970} +{"epoch": 0.86328125, "grad_norm": 0.2076922357082367, "learning_rate": 6.885053657779273e-05, "loss": 0.0015943592879921198, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.0016, "step": 221, "tokens/total": 7150848, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 100442} +{"epoch": 0.8671875, "grad_norm": 0.1208045557141304, "learning_rate": 6.857405174478604e-05, "loss": 0.0006645859684795141, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00066, "step": 222, "tokens/total": 7185176, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 100895} +{"epoch": 0.87109375, "grad_norm": 0.384154736995697, "learning_rate": 6.82970020400792e-05, "loss": 0.006838214583694935, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00686, "step": 223, "tokens/total": 7219024, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 101342} +{"epoch": 0.875, "grad_norm": 0.23179367184638977, "learning_rate": 6.801939899284132e-05, "loss": 0.010227427817881107, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.01028, "step": 224, "tokens/total": 7253816, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 101848} +{"epoch": 0.87890625, "grad_norm": 0.2610826790332794, "learning_rate": 6.774125415526827e-05, "loss": 0.0029095145873725414, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00291, "step": 225, "tokens/total": 7286344, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 102319} +{"epoch": 0.8828125, "grad_norm": 0.12811972200870514, "learning_rate": 6.746257910210214e-05, "loss": 0.0011515716323629022, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00115, "step": 226, "tokens/total": 7317952, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 102769} +{"epoch": 0.88671875, "grad_norm": 0.10018816590309143, "learning_rate": 6.718338543014937e-05, "loss": 0.0013016375014558434, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.0013, "step": 227, "tokens/total": 7347336, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 103219} +{"epoch": 0.890625, "grad_norm": 0.16356627643108368, "learning_rate": 6.69036847577983e-05, "loss": 0.0015832734061405063, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00158, "step": 228, "tokens/total": 7377592, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 103675} +{"epoch": 0.89453125, "grad_norm": 0.05339837446808815, "learning_rate": 6.662348872453553e-05, "loss": 0.0004287494230084121, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00043, "step": 229, "tokens/total": 7410184, "tokens/train_per_sec_per_gpu": 10.56, "tokens/trainable": 104136} +{"epoch": 0.8984375, "grad_norm": 0.24197232723236084, "learning_rate": 6.63428089904618e-05, "loss": 0.0009001469588838518, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.0009, "step": 230, "tokens/total": 7443984, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 104588} +{"epoch": 0.90234375, "grad_norm": 0.01966446079313755, "learning_rate": 6.60616572358065e-05, "loss": 0.0003684183757286519, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00037, "step": 231, "tokens/total": 7472480, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 104988} +{"epoch": 0.90625, "grad_norm": 0.09336209297180176, "learning_rate": 6.578004516044172e-05, "loss": 0.0010150084272027016, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00102, "step": 232, "tokens/total": 7505992, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 105455} +{"epoch": 0.91015625, "grad_norm": 0.019101327285170555, "learning_rate": 6.549798448339548e-05, "loss": 0.0002683868515305221, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00027, "step": 233, "tokens/total": 7541168, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 105918} +{"epoch": 0.9140625, "grad_norm": 0.9741220474243164, "learning_rate": 6.521548694236384e-05, "loss": 0.007494182325899601, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00752, "step": 234, "tokens/total": 7574208, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 106355} +{"epoch": 0.91796875, "grad_norm": 0.6059281229972839, "learning_rate": 6.493256429322259e-05, "loss": 0.0017166369361802936, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00172, "step": 235, "tokens/total": 7607016, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 106786} +{"epoch": 0.921875, "grad_norm": 0.26694199442863464, "learning_rate": 6.464922830953799e-05, "loss": 0.0005965263117104769, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.0006, "step": 236, "tokens/total": 7641744, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 107219} +{"epoch": 0.92578125, "grad_norm": 1.947953701019287, "learning_rate": 6.436549078207688e-05, "loss": 0.010355065576732159, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.01041, "step": 237, "tokens/total": 7673096, "tokens/train_per_sec_per_gpu": 10.16, "tokens/trainable": 107679} +{"epoch": 0.9296875, "grad_norm": 0.22162675857543945, "learning_rate": 6.408136351831592e-05, "loss": 0.0018127537332475185, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00181, "step": 238, "tokens/total": 7705344, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 108119} +{"epoch": 0.93359375, "grad_norm": 0.2628527581691742, "learning_rate": 6.379685834195036e-05, "loss": 0.002754376269876957, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00276, "step": 239, "tokens/total": 7740240, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 108626} +{"epoch": 0.9375, "grad_norm": 0.05739903450012207, "learning_rate": 6.351198709240186e-05, "loss": 0.0009274197509512305, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00093, "step": 240, "tokens/total": 7775392, "tokens/train_per_sec_per_gpu": 7.12, "tokens/trainable": 109079} +{"epoch": 0.94140625, "grad_norm": 0.9340605735778809, "learning_rate": 6.32267616243259e-05, "loss": 0.005681659560650587, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.0057, "step": 241, "tokens/total": 7807296, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 109542} +{"epoch": 0.9453125, "grad_norm": 0.33087393641471863, "learning_rate": 6.294119380711849e-05, "loss": 0.0012869058409705758, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00129, "step": 242, "tokens/total": 7838200, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 109969} +{"epoch": 0.94921875, "grad_norm": 0.2755827307701111, "learning_rate": 6.265529552442209e-05, "loss": 0.0019060819176957011, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00191, "step": 243, "tokens/total": 7870936, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 110431} +{"epoch": 0.953125, "grad_norm": 0.011986677534878254, "learning_rate": 6.236907867363127e-05, "loss": 7.013829599600285e-05, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00007, "step": 244, "tokens/total": 7902512, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 110877} +{"epoch": 0.95703125, "grad_norm": 0.03752681985497475, "learning_rate": 6.208255516539749e-05, "loss": 7.079268107190728e-05, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00007, "step": 245, "tokens/total": 7933224, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 111350} +{"epoch": 0.9609375, "grad_norm": 0.06505874544382095, "learning_rate": 6.179573692313344e-05, "loss": 0.00026803198852576315, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00027, "step": 246, "tokens/total": 7968456, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 111798} +{"epoch": 0.96484375, "grad_norm": 0.898012638092041, "learning_rate": 6.150863588251694e-05, "loss": 0.006320942658931017, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00634, "step": 247, "tokens/total": 8003592, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 112264} +{"epoch": 0.96875, "grad_norm": 1.3435603380203247, "learning_rate": 6.122126399099419e-05, "loss": 0.014099529944360256, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.0142, "step": 248, "tokens/total": 8040048, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 112772} +{"epoch": 0.97265625, "grad_norm": 0.013730109669268131, "learning_rate": 6.0933633207282615e-05, "loss": 6.4218977058772e-05, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00006, "step": 249, "tokens/total": 8074000, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 113278} +{"epoch": 0.9765625, "grad_norm": 0.2793673276901245, "learning_rate": 6.064575550087316e-05, "loss": 0.0013547725975513458, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00136, "step": 250, "tokens/total": 8108304, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 113716} +{"epoch": 0.98046875, "grad_norm": 0.07295020669698715, "learning_rate": 6.0357642851532245e-05, "loss": 0.00020697050786111504, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00021, "step": 251, "tokens/total": 8143464, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 114207} +{"epoch": 0.984375, "grad_norm": 0.5548952221870422, "learning_rate": 6.0069307248803294e-05, "loss": 0.010831203311681747, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.01089, "step": 252, "tokens/total": 8175576, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 114628} +{"epoch": 0.98828125, "grad_norm": 0.0007507039699703455, "learning_rate": 5.9780760691507635e-05, "loss": 1.693716512818355e-05, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00002, "step": 253, "tokens/total": 8205528, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 115060} +{"epoch": 0.9921875, "grad_norm": 0.8708575367927551, "learning_rate": 5.9492015187245334e-05, "loss": 0.008496318012475967, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00853, "step": 254, "tokens/total": 8239424, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 115514} +{"epoch": 0.99609375, "grad_norm": 0.009608994238078594, "learning_rate": 5.920308275189541e-05, "loss": 0.000111374567495659, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00011, "step": 255, "tokens/total": 8270392, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 115915} +{"epoch": 1.0, "grad_norm": 0.6723805069923401, "learning_rate": 5.8913975409115874e-05, "loss": 0.0029970293398946524, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.003, "step": 256, "tokens/total": 8302096, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 116378} +{"epoch": 1.00390625, "grad_norm": 0.22574518620967865, "learning_rate": 5.8624705189843395e-05, "loss": 0.0027854645159095526, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00279, "step": 257, "tokens/total": 8337064, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 116865} +{"epoch": 1.0078125, "grad_norm": 0.48540517687797546, "learning_rate": 5.833528413179249e-05, "loss": 0.007822631858289242, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00785, "step": 258, "tokens/total": 8372840, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 117363} +{"epoch": 1.01171875, "grad_norm": 0.20144635438919067, "learning_rate": 5.80457242789548e-05, "loss": 0.004275781102478504, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00428, "step": 259, "tokens/total": 8409728, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 117860} +{"epoch": 1.015625, "grad_norm": 0.9529743194580078, "learning_rate": 5.77560376810977e-05, "loss": 0.017187824472784996, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.01734, "step": 260, "tokens/total": 8442296, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 118324} +{"epoch": 1.01953125, "grad_norm": 0.061184775084257126, "learning_rate": 5.7466236393263005e-05, "loss": 0.0005474545760080218, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00055, "step": 261, "tokens/total": 8474568, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 118770} +{"epoch": 1.0234375, "grad_norm": 0.1571202576160431, "learning_rate": 5.717633247526522e-05, "loss": 0.001428767223842442, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00143, "step": 262, "tokens/total": 8505520, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 119206} +{"epoch": 1.02734375, "grad_norm": 0.04236381873488426, "learning_rate": 5.688633799118971e-05, "loss": 0.0005325078964233398, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00053, "step": 263, "tokens/total": 8535064, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 119610} +{"epoch": 1.03125, "grad_norm": 0.6967503428459167, "learning_rate": 5.659626500889066e-05, "loss": 0.009091313928365707, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00913, "step": 264, "tokens/total": 8563600, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 120080} +{"epoch": 1.03515625, "grad_norm": 0.14587803184986115, "learning_rate": 5.6306125599488905e-05, "loss": 0.0011287947418168187, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00113, "step": 265, "tokens/total": 8592648, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 120519} +{"epoch": 1.0390625, "grad_norm": 0.19905444979667664, "learning_rate": 5.601593183686955e-05, "loss": 0.004695571959018707, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00471, "step": 266, "tokens/total": 8627328, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 120995} +{"epoch": 1.04296875, "grad_norm": 0.176558718085289, "learning_rate": 5.572569579717961e-05, "loss": 0.0024751999881118536, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00248, "step": 267, "tokens/total": 8659760, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 121459} +{"epoch": 1.046875, "grad_norm": 0.030475091189146042, "learning_rate": 5.543542955832538e-05, "loss": 0.00036206768709234893, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00036, "step": 268, "tokens/total": 8696968, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 121902} +{"epoch": 1.05078125, "grad_norm": 0.4223625659942627, "learning_rate": 5.514514519946986e-05, "loss": 0.007860188372433186, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00789, "step": 269, "tokens/total": 8735392, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 122399} +{"epoch": 1.0546875, "grad_norm": 0.05047200620174408, "learning_rate": 5.485485480053015e-05, "loss": 0.0006843502051196992, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00068, "step": 270, "tokens/total": 8769840, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 122804} +{"epoch": 1.05859375, "grad_norm": 0.27951839566230774, "learning_rate": 5.4564570441674645e-05, "loss": 0.0034167547710239887, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00342, "step": 271, "tokens/total": 8800648, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 123244} +{"epoch": 1.0625, "grad_norm": 0.18229278922080994, "learning_rate": 5.42743042028204e-05, "loss": 0.002195717068389058, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.0022, "step": 272, "tokens/total": 8832480, "tokens/train_per_sec_per_gpu": 6.81, "tokens/trainable": 123683} +{"epoch": 1.06640625, "grad_norm": 0.07386018335819244, "learning_rate": 5.3984068163130464e-05, "loss": 0.001503078849054873, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.0015, "step": 273, "tokens/total": 8864840, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 124153} +{"epoch": 1.0703125, "grad_norm": 1.0502839088439941, "learning_rate": 5.369387440051111e-05, "loss": 0.000958785938564688, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00096, "step": 274, "tokens/total": 8897272, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 124616} +{"epoch": 1.07421875, "grad_norm": 0.04015328362584114, "learning_rate": 5.340373499110935e-05, "loss": 0.00030330964364111423, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.0003, "step": 275, "tokens/total": 8930688, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 125109} +{"epoch": 1.078125, "grad_norm": 0.02347392961382866, "learning_rate": 5.3113662008810304e-05, "loss": 0.00025518343318253756, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00026, "step": 276, "tokens/total": 8967216, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 125591} +{"epoch": 1.08203125, "grad_norm": 0.2118455022573471, "learning_rate": 5.282366752473479e-05, "loss": 0.002516010543331504, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00252, "step": 277, "tokens/total": 9001368, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 126065} +{"epoch": 1.0859375, "grad_norm": 0.09793723374605179, "learning_rate": 5.2533763606737005e-05, "loss": 0.00037439120933413506, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00037, "step": 278, "tokens/total": 9036808, "tokens/train_per_sec_per_gpu": 6.92, "tokens/trainable": 126480} +{"epoch": 1.08984375, "grad_norm": 0.12998050451278687, "learning_rate": 5.224396231890232e-05, "loss": 0.0008276088628917933, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00083, "step": 279, "tokens/total": 9067840, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 126929} +{"epoch": 1.09375, "grad_norm": 0.08307091891765594, "learning_rate": 5.195427572104522e-05, "loss": 0.0011488391319289804, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00115, "step": 280, "tokens/total": 9098936, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 127366} +{"epoch": 1.09765625, "grad_norm": 0.19091342389583588, "learning_rate": 5.166471586820751e-05, "loss": 0.0013439225731417537, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00134, "step": 281, "tokens/total": 9133264, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 127831} +{"epoch": 1.1015625, "grad_norm": 0.09817512333393097, "learning_rate": 5.1375294810156615e-05, "loss": 0.00095903011970222, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00096, "step": 282, "tokens/total": 9165008, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 128278} +{"epoch": 1.10546875, "grad_norm": 0.02531415782868862, "learning_rate": 5.1086024590884144e-05, "loss": 0.00025239313254132867, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00025, "step": 283, "tokens/total": 9197952, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 128727} +{"epoch": 1.109375, "grad_norm": 0.030117521062493324, "learning_rate": 5.079691724810461e-05, "loss": 0.00036268430994823575, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00036, "step": 284, "tokens/total": 9232576, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 129162} +{"epoch": 1.11328125, "grad_norm": 0.04540003836154938, "learning_rate": 5.0507984812754684e-05, "loss": 0.0004963502287864685, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.0005, "step": 285, "tokens/total": 9264568, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 129659} +{"epoch": 1.1171875, "grad_norm": 0.015218262560665607, "learning_rate": 5.021923930849237e-05, "loss": 0.00013353981194086373, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00013, "step": 286, "tokens/total": 9296544, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 130145} +{"epoch": 1.12109375, "grad_norm": 0.00775527348741889, "learning_rate": 4.99306927511967e-05, "loss": 7.387057121377438e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00007, "step": 287, "tokens/total": 9327936, "tokens/train_per_sec_per_gpu": 10.45, "tokens/trainable": 130585} +{"epoch": 1.125, "grad_norm": 0.015169062651693821, "learning_rate": 4.964235714846775e-05, "loss": 0.0001318678114330396, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00013, "step": 288, "tokens/total": 9357984, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 131005} +{"epoch": 1.12890625, "grad_norm": 0.08948688209056854, "learning_rate": 4.9354244499126866e-05, "loss": 0.000810101511888206, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00081, "step": 289, "tokens/total": 9388664, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 131471} +{"epoch": 1.1328125, "grad_norm": 0.003191933734342456, "learning_rate": 4.90663667927174e-05, "loss": 3.105036012129858e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00003, "step": 290, "tokens/total": 9420504, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 131920} +{"epoch": 1.13671875, "grad_norm": 0.11123140901327133, "learning_rate": 4.877873600900581e-05, "loss": 0.0005176699487492442, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00052, "step": 291, "tokens/total": 9452664, "tokens/train_per_sec_per_gpu": 6.31, "tokens/trainable": 132379} +{"epoch": 1.140625, "grad_norm": 0.004191562999039888, "learning_rate": 4.849136411748306e-05, "loss": 3.978127642767504e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00004, "step": 292, "tokens/total": 9488896, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 132851} +{"epoch": 1.14453125, "grad_norm": 0.18693070113658905, "learning_rate": 4.8204263076866574e-05, "loss": 0.0011224827030673623, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00112, "step": 293, "tokens/total": 9522112, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 133351} +{"epoch": 1.1484375, "grad_norm": 0.0030022314749658108, "learning_rate": 4.791744483460251e-05, "loss": 2.498782123439014e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00002, "step": 294, "tokens/total": 9555632, "tokens/train_per_sec_per_gpu": 6.6, "tokens/trainable": 133784} +{"epoch": 1.15234375, "grad_norm": 0.12991631031036377, "learning_rate": 4.7630921326368736e-05, "loss": 0.0005016294308006763, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.0005, "step": 295, "tokens/total": 9590904, "tokens/train_per_sec_per_gpu": 6.44, "tokens/trainable": 134209} +{"epoch": 1.15625, "grad_norm": 0.057920485734939575, "learning_rate": 4.7344704475577916e-05, "loss": 0.0004848130338359624, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00048, "step": 296, "tokens/total": 9620288, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 134645} +{"epoch": 1.16015625, "grad_norm": 0.8298271298408508, "learning_rate": 4.705880619288153e-05, "loss": 0.005292004439979792, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00531, "step": 297, "tokens/total": 9651984, "tokens/train_per_sec_per_gpu": 7.12, "tokens/trainable": 135047} +{"epoch": 1.1640625, "grad_norm": 0.02481858991086483, "learning_rate": 4.677323837567412e-05, "loss": 0.00013597604993265122, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00014, "step": 298, "tokens/total": 9681368, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 135474} +{"epoch": 1.16796875, "grad_norm": 0.007781519088894129, "learning_rate": 4.6488012907598146e-05, "loss": 3.305646532680839e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00003, "step": 299, "tokens/total": 9712872, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 135948} +{"epoch": 1.171875, "grad_norm": 0.46439334750175476, "learning_rate": 4.620314165804964e-05, "loss": 0.002267730189487338, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00227, "step": 300, "tokens/total": 9745184, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 136421} +{"epoch": 1.17578125, "grad_norm": 0.21304933726787567, "learning_rate": 4.591863648168407e-05, "loss": 0.0004081056686118245, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00041, "step": 301, "tokens/total": 9778928, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 136886} +{"epoch": 1.1796875, "grad_norm": 0.0856100395321846, "learning_rate": 4.5634509217923135e-05, "loss": 0.0003571913402993232, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00036, "step": 302, "tokens/total": 9810752, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 137321} +{"epoch": 1.18359375, "grad_norm": 0.44340500235557556, "learning_rate": 4.535077169046201e-05, "loss": 0.005811422597616911, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00583, "step": 303, "tokens/total": 9843672, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 137758} +{"epoch": 1.1875, "grad_norm": 0.009372885338962078, "learning_rate": 4.506743570677743e-05, "loss": 5.964743104414083e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00006, "step": 304, "tokens/total": 9873752, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 138190} +{"epoch": 1.19140625, "grad_norm": 0.43183693289756775, "learning_rate": 4.478451305763618e-05, "loss": 0.0053340112790465355, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00535, "step": 305, "tokens/total": 9906272, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 138650} +{"epoch": 1.1953125, "grad_norm": 0.0008696653530932963, "learning_rate": 4.450201551660454e-05, "loss": 9.230314390151761e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00001, "step": 306, "tokens/total": 9936296, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 139082} +{"epoch": 1.19921875, "grad_norm": 0.06367874890565872, "learning_rate": 4.4219954839558276e-05, "loss": 0.0002874258498195559, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00029, "step": 307, "tokens/total": 9967720, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 139523} +{"epoch": 1.203125, "grad_norm": 0.06051461771130562, "learning_rate": 4.393834276419352e-05, "loss": 0.00024053340894170105, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.00024, "step": 308, "tokens/total": 9997176, "tokens/train_per_sec_per_gpu": 10.84, "tokens/trainable": 140010} +{"epoch": 1.20703125, "grad_norm": 0.8396506309509277, "learning_rate": 4.36571910095382e-05, "loss": 0.005780715029686689, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.0058, "step": 309, "tokens/total": 10031872, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 140454} +{"epoch": 1.2109375, "grad_norm": 0.05915005877614021, "learning_rate": 4.337651127546448e-05, "loss": 0.00019831866666208953, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0002, "step": 310, "tokens/total": 10065240, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 140905} +{"epoch": 1.21484375, "grad_norm": 0.08531660586595535, "learning_rate": 4.3096315242201736e-05, "loss": 0.00038881110958755016, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00039, "step": 311, "tokens/total": 10096640, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 141339} +{"epoch": 1.21875, "grad_norm": 0.02239008992910385, "learning_rate": 4.2816614569850635e-05, "loss": 0.00012313105980865657, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00012, "step": 312, "tokens/total": 10130192, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 141778} +{"epoch": 1.22265625, "grad_norm": 0.9183381199836731, "learning_rate": 4.2537420897897864e-05, "loss": 0.005901237018406391, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00592, "step": 313, "tokens/total": 10164248, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 142255} +{"epoch": 1.2265625, "grad_norm": 0.03939570114016533, "learning_rate": 4.225874584473174e-05, "loss": 0.00021582888439297676, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00022, "step": 314, "tokens/total": 10194664, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 142693} +{"epoch": 1.23046875, "grad_norm": 0.45015043020248413, "learning_rate": 4.19806010071587e-05, "loss": 0.006441074423491955, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00646, "step": 315, "tokens/total": 10227768, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 143157} +{"epoch": 1.234375, "grad_norm": 0.06259039044380188, "learning_rate": 4.170299795992081e-05, "loss": 0.0005898751551285386, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00059, "step": 316, "tokens/total": 10260464, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 143603} +{"epoch": 1.23828125, "grad_norm": 0.00271618296392262, "learning_rate": 4.142594825521398e-05, "loss": 2.496850720490329e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00002, "step": 317, "tokens/total": 10297528, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 144084} +{"epoch": 1.2421875, "grad_norm": 0.008978031575679779, "learning_rate": 4.114946342220728e-05, "loss": 7.637684029759839e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00008, "step": 318, "tokens/total": 10331448, "tokens/train_per_sec_per_gpu": 7.0, "tokens/trainable": 144532} +{"epoch": 1.24609375, "grad_norm": 0.004866707604378462, "learning_rate": 4.087355496656321e-05, "loss": 5.6937213230412453e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00006, "step": 319, "tokens/total": 10363152, "tokens/train_per_sec_per_gpu": 10.19, "tokens/trainable": 145004} +{"epoch": 1.25, "grad_norm": 0.5986525416374207, "learning_rate": 4.05982343699588e-05, "loss": 0.0066916728392243385, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00671, "step": 320, "tokens/total": 10397832, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 145498} +{"epoch": 1.25390625, "grad_norm": 0.5054465532302856, "learning_rate": 4.0323513089607876e-05, "loss": 0.0023606501054018736, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00236, "step": 321, "tokens/total": 10431416, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 145965} +{"epoch": 1.2578125, "grad_norm": 1.2171269655227661, "learning_rate": 4.004940255778431e-05, "loss": 0.008329451084136963, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00836, "step": 322, "tokens/total": 10467384, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 146409} +{"epoch": 1.26171875, "grad_norm": 0.013993712142109871, "learning_rate": 3.977591418134619e-05, "loss": 0.00018414166697766632, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00018, "step": 323, "tokens/total": 10500792, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 146849} +{"epoch": 1.265625, "grad_norm": 0.0054016923531889915, "learning_rate": 3.95030593412612e-05, "loss": 5.7090208429144695e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00006, "step": 324, "tokens/total": 10532592, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 147272} +{"epoch": 1.26953125, "grad_norm": 0.43134722113609314, "learning_rate": 3.923084939213296e-05, "loss": 0.004804352764040232, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00482, "step": 325, "tokens/total": 10562392, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 147694} +{"epoch": 1.2734375, "grad_norm": 0.024546338245272636, "learning_rate": 3.895929566172861e-05, "loss": 0.00019788103236351162, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.0002, "step": 326, "tokens/total": 10595416, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 148158} +{"epoch": 1.27734375, "grad_norm": 0.08384862542152405, "learning_rate": 3.868840945050728e-05, "loss": 0.0010594242485240102, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00106, "step": 327, "tokens/total": 10630504, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 148590} +{"epoch": 1.28125, "grad_norm": 0.09186010807752609, "learning_rate": 3.841820203114995e-05, "loss": 0.0015117102302610874, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00151, "step": 328, "tokens/total": 10661632, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 149037} +{"epoch": 1.28515625, "grad_norm": 0.4388459622859955, "learning_rate": 3.814868464809027e-05, "loss": 0.006318775936961174, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00634, "step": 329, "tokens/total": 10691544, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 149439} +{"epoch": 1.2890625, "grad_norm": 0.33691108226776123, "learning_rate": 3.787986851704667e-05, "loss": 0.006742274854332209, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00677, "step": 330, "tokens/total": 10724768, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 149869} +{"epoch": 1.29296875, "grad_norm": 0.17683067917823792, "learning_rate": 3.7611764824555654e-05, "loss": 0.0013654412468895316, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00137, "step": 331, "tokens/total": 10756008, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 150343} +{"epoch": 1.296875, "grad_norm": 0.019201505929231644, "learning_rate": 3.734438472750619e-05, "loss": 0.00017084932187572122, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00017, "step": 332, "tokens/total": 10793152, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 150814} +{"epoch": 1.30078125, "grad_norm": 0.05624443292617798, "learning_rate": 3.707773935267552e-05, "loss": 0.00017720855248626322, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00018, "step": 333, "tokens/total": 10827624, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 151300} +{"epoch": 1.3046875, "grad_norm": 0.004741082899272442, "learning_rate": 3.68118397962661e-05, "loss": 4.08182677347213e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00004, "step": 334, "tokens/total": 10859712, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 151708} +{"epoch": 1.30859375, "grad_norm": 0.1925305277109146, "learning_rate": 3.654669712344384e-05, "loss": 0.0017948064487427473, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.0018, "step": 335, "tokens/total": 10888792, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 152164} +{"epoch": 1.3125, "grad_norm": 0.031003044918179512, "learning_rate": 3.628232236787763e-05, "loss": 0.0002224243216915056, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00022, "step": 336, "tokens/total": 10922968, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 152603} +{"epoch": 1.31640625, "grad_norm": 0.019595874473452568, "learning_rate": 3.6018726531280144e-05, "loss": 0.00015013912343420088, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00015, "step": 337, "tokens/total": 10958000, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 153094} +{"epoch": 1.3203125, "grad_norm": 0.01442669052630663, "learning_rate": 3.575592058295017e-05, "loss": 0.00014124851441010833, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00014, "step": 338, "tokens/total": 10989872, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 153573} +{"epoch": 1.32421875, "grad_norm": 0.02336510457098484, "learning_rate": 3.549391545931585e-05, "loss": 0.0001512402668595314, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00015, "step": 339, "tokens/total": 11020784, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 154033} +{"epoch": 1.328125, "grad_norm": 0.1601199507713318, "learning_rate": 3.5232722063479914e-05, "loss": 0.0019681432750076056, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00197, "step": 340, "tokens/total": 11053240, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 154482} +{"epoch": 1.33203125, "grad_norm": 0.09220559149980545, "learning_rate": 3.49723512647657e-05, "loss": 0.0006545006763190031, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00065, "step": 341, "tokens/total": 11089040, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 154982} +{"epoch": 1.3359375, "grad_norm": 0.13310487568378448, "learning_rate": 3.471281389826491e-05, "loss": 0.0013007798697799444, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.0013, "step": 342, "tokens/total": 11119672, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 155387} +{"epoch": 1.33984375, "grad_norm": 0.02399330772459507, "learning_rate": 3.4454120764386764e-05, "loss": 0.00014109320181887597, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00014, "step": 343, "tokens/total": 11155040, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 155842} +{"epoch": 1.34375, "grad_norm": 0.02785874344408512, "learning_rate": 3.4196282628408526e-05, "loss": 0.0001045015305862762, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.0001, "step": 344, "tokens/total": 11185952, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 156287} +{"epoch": 1.34765625, "grad_norm": 0.02984408102929592, "learning_rate": 3.3939310220027456e-05, "loss": 0.00026802229695022106, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00027, "step": 345, "tokens/total": 11219272, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 156782} +{"epoch": 1.3515625, "grad_norm": 0.0967763289809227, "learning_rate": 3.3683214232914404e-05, "loss": 0.00020352053979877383, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.0002, "step": 346, "tokens/total": 11253424, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 157255} +{"epoch": 1.35546875, "grad_norm": 1.241949439048767, "learning_rate": 3.342800532426873e-05, "loss": 0.018477020785212517, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01865, "step": 347, "tokens/total": 11286216, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 157721} +{"epoch": 1.359375, "grad_norm": 0.007952549494802952, "learning_rate": 3.317369411437484e-05, "loss": 0.00010208190360572189, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.0001, "step": 348, "tokens/total": 11319480, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 158189} +{"epoch": 1.36328125, "grad_norm": 0.01419889833778143, "learning_rate": 3.292029118616024e-05, "loss": 0.00019108074775431305, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00019, "step": 349, "tokens/total": 11350472, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 158612} +{"epoch": 1.3671875, "grad_norm": 0.07352183759212494, "learning_rate": 3.266780708475511e-05, "loss": 0.0011991825886070728, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.0012, "step": 350, "tokens/total": 11383088, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 159090} +{"epoch": 1.37109375, "grad_norm": 0.0046110874973237514, "learning_rate": 3.241625231705354e-05, "loss": 8.018541848286986e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00008, "step": 351, "tokens/total": 11418392, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 159549} +{"epoch": 1.375, "grad_norm": 0.004159008618444204, "learning_rate": 3.216563735127618e-05, "loss": 6.694348121527582e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00007, "step": 352, "tokens/total": 11449784, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 160026} +{"epoch": 1.37890625, "grad_norm": 0.20275245606899261, "learning_rate": 3.191597261653475e-05, "loss": 0.003815322881564498, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00382, "step": 353, "tokens/total": 11484240, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 160487} +{"epoch": 1.3828125, "grad_norm": 0.014782947488129139, "learning_rate": 3.166726850239794e-05, "loss": 0.0001291236258111894, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00013, "step": 354, "tokens/total": 11513360, "tokens/train_per_sec_per_gpu": 11.04, "tokens/trainable": 160935} +{"epoch": 1.38671875, "grad_norm": 0.053277432918548584, "learning_rate": 3.141953535845912e-05, "loss": 0.0003319506940897554, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00033, "step": 355, "tokens/total": 11543112, "tokens/train_per_sec_per_gpu": 10.3, "tokens/trainable": 161385} +{"epoch": 1.390625, "grad_norm": 0.017092958092689514, "learning_rate": 3.11727834939056e-05, "loss": 0.0002632134419400245, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00026, "step": 356, "tokens/total": 11573464, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 161849} +{"epoch": 1.39453125, "grad_norm": 0.13390351831912994, "learning_rate": 3.092702317708967e-05, "loss": 0.0008296740124933422, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00083, "step": 357, "tokens/total": 11604896, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 162306} +{"epoch": 1.3984375, "grad_norm": 0.017895657569169998, "learning_rate": 3.0682264635101276e-05, "loss": 0.00020070478785783052, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.0002, "step": 358, "tokens/total": 11636368, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 162759} +{"epoch": 1.40234375, "grad_norm": 0.1778826117515564, "learning_rate": 3.0438518053342407e-05, "loss": 0.0014261718606576324, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00143, "step": 359, "tokens/total": 11670216, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 163230} +{"epoch": 1.40625, "grad_norm": 0.05375527963042259, "learning_rate": 3.0195793575103266e-05, "loss": 0.00041517114732414484, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00042, "step": 360, "tokens/total": 11707816, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 163694} +{"epoch": 1.41015625, "grad_norm": 0.10273242741823196, "learning_rate": 2.9954101301140146e-05, "loss": 0.0010008697863668203, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.001, "step": 361, "tokens/total": 11742584, "tokens/train_per_sec_per_gpu": 6.93, "tokens/trainable": 164144} +{"epoch": 1.4140625, "grad_norm": 0.07318515330553055, "learning_rate": 2.9713451289255123e-05, "loss": 0.0005194327677600086, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00052, "step": 362, "tokens/total": 11770384, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 164585} +{"epoch": 1.41796875, "grad_norm": 0.006921444553881884, "learning_rate": 2.9473853553877484e-05, "loss": 0.00010943078086711466, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00011, "step": 363, "tokens/total": 11803208, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 165059} +{"epoch": 1.421875, "grad_norm": 0.10747949779033661, "learning_rate": 2.9235318065647e-05, "loss": 0.0012156660668551922, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00122, "step": 364, "tokens/total": 11840480, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 165519} +{"epoch": 1.42578125, "grad_norm": 0.0010617281077429652, "learning_rate": 2.8997854750998964e-05, "loss": 1.657361826801207e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00002, "step": 365, "tokens/total": 11875544, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 165984} +{"epoch": 1.4296875, "grad_norm": 0.012131115421652794, "learning_rate": 2.8761473491751258e-05, "loss": 0.00010090768773807213, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.0001, "step": 366, "tokens/total": 11907656, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 166419} +{"epoch": 1.43359375, "grad_norm": 0.017483556643128395, "learning_rate": 2.8526184124692883e-05, "loss": 0.00016465615772176534, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00016, "step": 367, "tokens/total": 11941904, "tokens/train_per_sec_per_gpu": 6.42, "tokens/trainable": 166857} +{"epoch": 1.4375, "grad_norm": 0.37556013464927673, "learning_rate": 2.829199644117484e-05, "loss": 0.006614780053496361, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00664, "step": 368, "tokens/total": 11973624, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 167267} +{"epoch": 1.44140625, "grad_norm": 0.038173969835042953, "learning_rate": 2.8058920186702553e-05, "loss": 5.801235238322988e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00006, "step": 369, "tokens/total": 12010360, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 167740} +{"epoch": 1.4453125, "grad_norm": 0.008688279427587986, "learning_rate": 2.782696506053033e-05, "loss": 5.0976879720110446e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00005, "step": 370, "tokens/total": 12044544, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 168172} +{"epoch": 1.44921875, "grad_norm": 0.05912031978368759, "learning_rate": 2.7596140715257824e-05, "loss": 0.0004299452994018793, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00043, "step": 371, "tokens/total": 12075792, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 168637} +{"epoch": 1.453125, "grad_norm": 0.02544698305428028, "learning_rate": 2.7366456756428184e-05, "loss": 0.0001728852221276611, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00017, "step": 372, "tokens/total": 12108088, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 169094} +{"epoch": 1.45703125, "grad_norm": 0.017548788338899612, "learning_rate": 2.7137922742128486e-05, "loss": 0.0001250207715202123, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00013, "step": 373, "tokens/total": 12140888, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 169593} +{"epoch": 1.4609375, "grad_norm": 0.0018757880898192525, "learning_rate": 2.691054818259188e-05, "loss": 3.251901944167912e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00003, "step": 374, "tokens/total": 12174560, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 170066} +{"epoch": 1.46484375, "grad_norm": 0.002028173301368952, "learning_rate": 2.6684342539801933e-05, "loss": 3.217678386135958e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00003, "step": 375, "tokens/total": 12204848, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 170518} +{"epoch": 1.46875, "grad_norm": 0.02725907601416111, "learning_rate": 2.645931522709877e-05, "loss": 0.00020808764384128153, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00021, "step": 376, "tokens/total": 12238472, "tokens/train_per_sec_per_gpu": 10.81, "tokens/trainable": 170998} +{"epoch": 1.47265625, "grad_norm": 0.11016661673784256, "learning_rate": 2.6235475608787365e-05, "loss": 0.0008460779208689928, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00085, "step": 377, "tokens/total": 12271024, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 171459} +{"epoch": 1.4765625, "grad_norm": 0.08201520144939423, "learning_rate": 2.6012832999747916e-05, "loss": 0.0007661516428925097, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00077, "step": 378, "tokens/total": 12301312, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 171905} +{"epoch": 1.48046875, "grad_norm": 0.30397552251815796, "learning_rate": 2.579139666504821e-05, "loss": 0.007759178057312965, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00779, "step": 379, "tokens/total": 12337072, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 172351} +{"epoch": 1.484375, "grad_norm": 0.003901650430634618, "learning_rate": 2.557117581955798e-05, "loss": 3.616443063947372e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00004, "step": 380, "tokens/total": 12368000, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 172817} +{"epoch": 1.48828125, "grad_norm": 0.18711234629154205, "learning_rate": 2.5352179627565532e-05, "loss": 0.00125799176748842, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00126, "step": 381, "tokens/total": 12399880, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 173258} +{"epoch": 1.4921875, "grad_norm": 0.008143090642988682, "learning_rate": 2.5134417202396277e-05, "loss": 4.9053156544687226e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00005, "step": 382, "tokens/total": 12432288, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 173665} +{"epoch": 1.49609375, "grad_norm": 0.04150390625, "learning_rate": 2.491789760603361e-05, "loss": 0.0004025471571367234, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.0004, "step": 383, "tokens/total": 12465592, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 174095} +{"epoch": 1.5, "grad_norm": 0.016370786353945732, "learning_rate": 2.4702629848741764e-05, "loss": 0.0001135662678279914, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00011, "step": 384, "tokens/total": 12498200, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 174550} +{"epoch": 1.50390625, "grad_norm": 0.06813102960586548, "learning_rate": 2.4488622888690785e-05, "loss": 0.0006072627147659659, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00061, "step": 385, "tokens/total": 12530280, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 174990} +{"epoch": 1.5078125, "grad_norm": 0.017209941521286964, "learning_rate": 2.427588563158384e-05, "loss": 0.00016635535575915128, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00017, "step": 386, "tokens/total": 12563360, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 175476} +{"epoch": 1.51171875, "grad_norm": 0.07096823304891586, "learning_rate": 2.406442693028651e-05, "loss": 0.00034378794953227043, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00034, "step": 387, "tokens/total": 12593032, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 175897} +{"epoch": 1.515625, "grad_norm": 0.023032967001199722, "learning_rate": 2.3854255584458547e-05, "loss": 0.00022963191440794617, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00023, "step": 388, "tokens/total": 12628272, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 176324} +{"epoch": 1.51953125, "grad_norm": 0.001670168829150498, "learning_rate": 2.3645380340187508e-05, "loss": 2.594987745396793e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00003, "step": 389, "tokens/total": 12661368, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 176805} +{"epoch": 1.5234375, "grad_norm": 0.23060372471809387, "learning_rate": 2.3437809889624914e-05, "loss": 0.00214630039408803, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.77, "memory/max_allocated (GiB)": 63.77, "ppl": 1.00215, "step": 390, "tokens/total": 12691168, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 177256} +{"epoch": 1.52734375, "grad_norm": 0.0027266168035566807, "learning_rate": 2.3231552870624487e-05, "loss": 3.107040538452566e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00003, "step": 391, "tokens/total": 12721264, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 177682} +{"epoch": 1.53125, "grad_norm": 0.01850302144885063, "learning_rate": 2.3026617866382657e-05, "loss": 0.00014721925253979862, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00015, "step": 392, "tokens/total": 12753640, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 178179} +{"epoch": 1.53515625, "grad_norm": 0.002697126939892769, "learning_rate": 2.2823013405081507e-05, "loss": 3.0051523935981095e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00003, "step": 393, "tokens/total": 12787056, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 178676} +{"epoch": 1.5390625, "grad_norm": 0.001310309860855341, "learning_rate": 2.2620747959533722e-05, "loss": 2.6425068426760845e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00003, "step": 394, "tokens/total": 12819312, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 179173} +{"epoch": 1.54296875, "grad_norm": 0.004896394442766905, "learning_rate": 2.2419829946830123e-05, "loss": 4.779961454914883e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00005, "step": 395, "tokens/total": 12853376, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 179624} +{"epoch": 1.546875, "grad_norm": 0.035782340914011, "learning_rate": 2.2220267727989325e-05, "loss": 0.00035034213215112686, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00035, "step": 396, "tokens/total": 12882960, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 180078} +{"epoch": 1.55078125, "grad_norm": 0.08266805112361908, "learning_rate": 2.202206960760984e-05, "loss": 0.0007082950905896723, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00071, "step": 397, "tokens/total": 12917016, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 180585} +{"epoch": 1.5546875, "grad_norm": 0.041303567588329315, "learning_rate": 2.182524383352446e-05, "loss": 0.0003182778600603342, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00032, "step": 398, "tokens/total": 12950056, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 181039} +{"epoch": 1.55859375, "grad_norm": 0.04584497585892677, "learning_rate": 2.1629798596457056e-05, "loss": 0.0002194482076447457, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00022, "step": 399, "tokens/total": 12981728, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 181489} +{"epoch": 1.5625, "grad_norm": 0.03409041464328766, "learning_rate": 2.1435742029681725e-05, "loss": 0.00015452716615982354, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00015, "step": 400, "tokens/total": 13011464, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 181906} +{"epoch": 1.56640625, "grad_norm": 0.001649749930948019, "learning_rate": 2.124308220868431e-05, "loss": 2.2223175619728863e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00002, "step": 401, "tokens/total": 13044432, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 182361} +{"epoch": 1.5703125, "grad_norm": 0.004292014986276627, "learning_rate": 2.105182715082638e-05, "loss": 3.402712172828615e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00003, "step": 402, "tokens/total": 13075608, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 182838} +{"epoch": 1.57421875, "grad_norm": 0.008646000176668167, "learning_rate": 2.0861984815011552e-05, "loss": 4.6871460654074326e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00005, "step": 403, "tokens/total": 13107352, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 183269} +{"epoch": 1.578125, "grad_norm": 0.0065473392605781555, "learning_rate": 2.0673563101354323e-05, "loss": 2.1935953554930165e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00002, "step": 404, "tokens/total": 13140736, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 183699} +{"epoch": 1.58203125, "grad_norm": 0.01355015765875578, "learning_rate": 2.0486569850851317e-05, "loss": 0.00010233926877845079, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.0001, "step": 405, "tokens/total": 13174040, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 184162} +{"epoch": 1.5859375, "grad_norm": 0.004683691076934338, "learning_rate": 2.0301012845054956e-05, "loss": 5.172019882593304e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00005, "step": 406, "tokens/total": 13207232, "tokens/train_per_sec_per_gpu": 6.6, "tokens/trainable": 184576} +{"epoch": 1.58984375, "grad_norm": 0.009543081745505333, "learning_rate": 2.011689980574966e-05, "loss": 5.5293741752393544e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00006, "step": 407, "tokens/total": 13241160, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 184995} +{"epoch": 1.59375, "grad_norm": 0.13632792234420776, "learning_rate": 1.993423839463052e-05, "loss": 0.00048487362801097333, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00048, "step": 408, "tokens/total": 13273104, "tokens/train_per_sec_per_gpu": 10.47, "tokens/trainable": 185452} +{"epoch": 1.59765625, "grad_norm": 0.00267207995057106, "learning_rate": 1.975303621298445e-05, "loss": 4.730775617645122e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00005, "step": 409, "tokens/total": 13308136, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 185904} +{"epoch": 1.6015625, "grad_norm": 0.008396605961024761, "learning_rate": 1.957330080137385e-05, "loss": 4.3605272367130965e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00004, "step": 410, "tokens/total": 13337816, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 186355} +{"epoch": 1.60546875, "grad_norm": 0.002577191451564431, "learning_rate": 1.9395039639322864e-05, "loss": 1.9998911739094183e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00002, "step": 411, "tokens/total": 13372576, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 186831} +{"epoch": 1.609375, "grad_norm": 0.0010217571398243308, "learning_rate": 1.9218260145006073e-05, "loss": 1.206532760988921e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00001, "step": 412, "tokens/total": 13406664, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 187317} +{"epoch": 1.61328125, "grad_norm": 0.05137507990002632, "learning_rate": 1.904296967493982e-05, "loss": 0.00016286612662952393, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00016, "step": 413, "tokens/total": 13438256, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 187773} +{"epoch": 1.6171875, "grad_norm": 0.21306100487709045, "learning_rate": 1.8869175523676064e-05, "loss": 0.0016196463257074356, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00162, "step": 414, "tokens/total": 13472096, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 188268} +{"epoch": 1.62109375, "grad_norm": 0.011531089432537556, "learning_rate": 1.869688492349885e-05, "loss": 0.00010906728857662529, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00011, "step": 415, "tokens/total": 13504192, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 188720} +{"epoch": 1.625, "grad_norm": 0.7135410308837891, "learning_rate": 1.85261050441233e-05, "loss": 0.017678091302514076, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.01784, "step": 416, "tokens/total": 13537392, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 189177} +{"epoch": 1.62890625, "grad_norm": 0.37908095121383667, "learning_rate": 1.8356842992397304e-05, "loss": 0.003851323388516903, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00386, "step": 417, "tokens/total": 13569344, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 189617} +{"epoch": 1.6328125, "grad_norm": 0.054958634078502655, "learning_rate": 1.8189105812005714e-05, "loss": 0.00032354131690226495, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00032, "step": 418, "tokens/total": 13602200, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 190028} +{"epoch": 1.63671875, "grad_norm": 0.007500923704355955, "learning_rate": 1.802290048317732e-05, "loss": 6.821894930908456e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00007, "step": 419, "tokens/total": 13632640, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 190445} +{"epoch": 1.640625, "grad_norm": 0.005888704676181078, "learning_rate": 1.785823392239424e-05, "loss": 7.395091961370781e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00007, "step": 420, "tokens/total": 13664920, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 190886} +{"epoch": 1.64453125, "grad_norm": 0.0038128895685076714, "learning_rate": 1.7695112982104225e-05, "loss": 2.993961243191734e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00003, "step": 421, "tokens/total": 13697144, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 191325} +{"epoch": 1.6484375, "grad_norm": 0.0042328559793531895, "learning_rate": 1.7533544450435433e-05, "loss": 2.1969190129311755e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00002, "step": 422, "tokens/total": 13724496, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 191758} +{"epoch": 1.65234375, "grad_norm": 0.0021966206841170788, "learning_rate": 1.7373535050913946e-05, "loss": 1.1232285032747313e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00001, "step": 423, "tokens/total": 13757880, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 192195} +{"epoch": 1.65625, "grad_norm": 0.08862802386283875, "learning_rate": 1.721509144218405e-05, "loss": 0.00043068130617029965, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00043, "step": 424, "tokens/total": 13787648, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 192625} +{"epoch": 1.66015625, "grad_norm": 0.14186088740825653, "learning_rate": 1.705822021773101e-05, "loss": 0.0012930360389873385, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00129, "step": 425, "tokens/total": 13819720, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 193111} +{"epoch": 1.6640625, "grad_norm": 0.11549319326877594, "learning_rate": 1.69029279056068e-05, "loss": 0.0005924305878579617, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00059, "step": 426, "tokens/total": 13854944, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 193551} +{"epoch": 1.66796875, "grad_norm": 0.3803383409976959, "learning_rate": 1.6749220968158415e-05, "loss": 0.000711844302713871, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00071, "step": 427, "tokens/total": 13887824, "tokens/train_per_sec_per_gpu": 10.4, "tokens/trainable": 194057} +{"epoch": 1.671875, "grad_norm": 0.1308572143316269, "learning_rate": 1.659710580175893e-05, "loss": 0.001282603945583105, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00128, "step": 428, "tokens/total": 13918192, "tokens/train_per_sec_per_gpu": 10.17, "tokens/trainable": 194556} +{"epoch": 1.67578125, "grad_norm": 0.0025182166136801243, "learning_rate": 1.644658873654133e-05, "loss": 3.724018461070955e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00004, "step": 429, "tokens/total": 13947440, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 194993} +{"epoch": 1.6796875, "grad_norm": 0.000528821547050029, "learning_rate": 1.629767603613508e-05, "loss": 1.3657980161951855e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00001, "step": 430, "tokens/total": 13974360, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 195393} +{"epoch": 1.68359375, "grad_norm": 0.007422046735882759, "learning_rate": 1.615037389740547e-05, "loss": 4.150793392909691e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00004, "step": 431, "tokens/total": 14006592, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 195837} +{"epoch": 1.6875, "grad_norm": 0.03179296478629112, "learning_rate": 1.600468845019576e-05, "loss": 0.00012934922415297478, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00013, "step": 432, "tokens/total": 14040776, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 196317} +{"epoch": 1.69140625, "grad_norm": 0.0007831910043023527, "learning_rate": 1.5860625757072092e-05, "loss": 1.7020538507495075e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00002, "step": 433, "tokens/total": 14073040, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 196765} +{"epoch": 1.6953125, "grad_norm": 0.0008605247712694108, "learning_rate": 1.571819181307116e-05, "loss": 1.18840835057199e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00001, "step": 434, "tokens/total": 14107624, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 197237} +{"epoch": 1.69921875, "grad_norm": 0.005227370653301477, "learning_rate": 1.557739254545075e-05, "loss": 3.961667243856937e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00004, "step": 435, "tokens/total": 14137784, "tokens/train_per_sec_per_gpu": 10.65, "tokens/trainable": 197709} +{"epoch": 1.703125, "grad_norm": 0.023645393550395966, "learning_rate": 1.543823381344311e-05, "loss": 0.00016329510253854096, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00016, "step": 436, "tokens/total": 14167776, "tokens/train_per_sec_per_gpu": 10.8, "tokens/trainable": 198184} +{"epoch": 1.70703125, "grad_norm": 0.12368526309728622, "learning_rate": 1.5300721408011114e-05, "loss": 0.00047180414549075067, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00047, "step": 437, "tokens/total": 14199752, "tokens/train_per_sec_per_gpu": 10.26, "tokens/trainable": 198641} +{"epoch": 1.7109375, "grad_norm": 0.0003165006055496633, "learning_rate": 1.5164861051607254e-05, "loss": 9.807238711800892e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.00001, "step": 438, "tokens/total": 14230344, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 199049} +{"epoch": 1.71484375, "grad_norm": 0.001645191223360598, "learning_rate": 1.5030658397935521e-05, "loss": 1.8102466128766537e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00002, "step": 439, "tokens/total": 14263808, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 199518} +{"epoch": 1.71875, "grad_norm": 0.013035300187766552, "learning_rate": 1.4898119031716104e-05, "loss": 3.8625759771093726e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00004, "step": 440, "tokens/total": 14295864, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 199976} +{"epoch": 1.72265625, "grad_norm": 0.023405559360980988, "learning_rate": 1.476724846845306e-05, "loss": 0.00013188720913603902, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00013, "step": 441, "tokens/total": 14331688, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 200453} +{"epoch": 1.7265625, "grad_norm": 0.002955840667709708, "learning_rate": 1.463805215420471e-05, "loss": 3.862907033180818e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00004, "step": 442, "tokens/total": 14364688, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 200923} +{"epoch": 1.73046875, "grad_norm": 0.0015322272665798664, "learning_rate": 1.451053546535705e-05, "loss": 1.7777541870600544e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00002, "step": 443, "tokens/total": 14397832, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 201379} +{"epoch": 1.734375, "grad_norm": 0.07097112387418747, "learning_rate": 1.438470370840001e-05, "loss": 0.00031711344490759075, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00032, "step": 444, "tokens/total": 14431864, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 201851} +{"epoch": 1.73828125, "grad_norm": 0.020527498796582222, "learning_rate": 1.4260562119706606e-05, "loss": 9.162687638308853e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00009, "step": 445, "tokens/total": 14462152, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 202325} +{"epoch": 1.7421875, "grad_norm": 0.16473887860774994, "learning_rate": 1.413811586531508e-05, "loss": 0.0010149714071303606, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00102, "step": 446, "tokens/total": 14493072, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 202766} +{"epoch": 1.74609375, "grad_norm": 0.2728119194507599, "learning_rate": 1.4017370040713884e-05, "loss": 0.000959982571657747, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00096, "step": 447, "tokens/total": 14527392, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 203215} +{"epoch": 1.75, "grad_norm": 0.16391518712043762, "learning_rate": 1.3898329670629645e-05, "loss": 0.0016111229779198766, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00161, "step": 448, "tokens/total": 14561992, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 203660} +{"epoch": 1.75390625, "grad_norm": 0.001078295987099409, "learning_rate": 1.3780999708818058e-05, "loss": 1.8065813492285088e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00002, "step": 449, "tokens/total": 14592056, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 204110} +{"epoch": 1.7578125, "grad_norm": 0.02322155050933361, "learning_rate": 1.3665385037857758e-05, "loss": 4.9721813411451876e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00005, "step": 450, "tokens/total": 14622816, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 204562} +{"epoch": 1.76171875, "grad_norm": 0.006959832273423672, "learning_rate": 1.3551490468947126e-05, "loss": 7.724652823526412e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00008, "step": 451, "tokens/total": 14653272, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 205008} +{"epoch": 1.765625, "grad_norm": 0.012051090598106384, "learning_rate": 1.3439320741704075e-05, "loss": 7.415656000375748e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00007, "step": 452, "tokens/total": 14683952, "tokens/train_per_sec_per_gpu": 10.05, "tokens/trainable": 205432} +{"epoch": 1.76953125, "grad_norm": 0.0051918332464993, "learning_rate": 1.3328880523968808e-05, "loss": 4.139387601753697e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00004, "step": 453, "tokens/total": 14716256, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 205895} +{"epoch": 1.7734375, "grad_norm": 0.006615712307393551, "learning_rate": 1.3220174411609587e-05, "loss": 5.309102562023327e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00005, "step": 454, "tokens/total": 14749088, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 206334} +{"epoch": 1.77734375, "grad_norm": 0.01709478721022606, "learning_rate": 1.3113206928331471e-05, "loss": 0.00011454458581283689, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00011, "step": 455, "tokens/total": 14777384, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 206796} +{"epoch": 1.78125, "grad_norm": 0.0024607751984149218, "learning_rate": 1.300798252548806e-05, "loss": 2.606587077025324e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00003, "step": 456, "tokens/total": 14809640, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 207248} +{"epoch": 1.78515625, "grad_norm": 0.1416245847940445, "learning_rate": 1.2904505581896265e-05, "loss": 0.0008817149791866541, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00088, "step": 457, "tokens/total": 14843728, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 207721} +{"epoch": 1.7890625, "grad_norm": 0.021349113434553146, "learning_rate": 1.2802780403654082e-05, "loss": 0.00017104606376960874, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00017, "step": 458, "tokens/total": 14876256, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 208168} +{"epoch": 1.79296875, "grad_norm": 0.012080291286110878, "learning_rate": 1.2702811223961408e-05, "loss": 0.0001241258578374982, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00012, "step": 459, "tokens/total": 14907000, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 208629} +{"epoch": 1.796875, "grad_norm": 0.016672659665346146, "learning_rate": 1.2604602202943861e-05, "loss": 3.6020359402755275e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00004, "step": 460, "tokens/total": 14937704, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 209073} +{"epoch": 1.80078125, "grad_norm": 0.019530391320586205, "learning_rate": 1.2508157427479686e-05, "loss": 8.726805390324444e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00009, "step": 461, "tokens/total": 14966920, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 209520} +{"epoch": 1.8046875, "grad_norm": 0.5205325484275818, "learning_rate": 1.2413480911029655e-05, "loss": 0.0012404883746057749, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00124, "step": 462, "tokens/total": 14996040, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 209958} +{"epoch": 1.80859375, "grad_norm": 0.0896746963262558, "learning_rate": 1.2320576593470082e-05, "loss": 0.0004465134989004582, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00045, "step": 463, "tokens/total": 15029096, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 210385} +{"epoch": 1.8125, "grad_norm": 0.005535534583032131, "learning_rate": 1.2229448340928828e-05, "loss": 4.538158100331202e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00005, "step": 464, "tokens/total": 15063696, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 210893} +{"epoch": 1.81640625, "grad_norm": 0.004458460491150618, "learning_rate": 1.2140099945624458e-05, "loss": 2.0305807993281633e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00002, "step": 465, "tokens/total": 15098264, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 211350} +{"epoch": 1.8203125, "grad_norm": 0.018291551619768143, "learning_rate": 1.205253512570841e-05, "loss": 0.00010985799599438906, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00011, "step": 466, "tokens/total": 15130464, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 211798} +{"epoch": 1.82421875, "grad_norm": 0.04486475884914398, "learning_rate": 1.1966757525110255e-05, "loss": 0.00027667643735185266, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00028, "step": 467, "tokens/total": 15158736, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 212249} +{"epoch": 1.828125, "grad_norm": 0.0777636244893074, "learning_rate": 1.1882770713386095e-05, "loss": 0.0005922932177782059, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00059, "step": 468, "tokens/total": 15193648, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 212724} +{"epoch": 1.83203125, "grad_norm": 0.016397234052419662, "learning_rate": 1.180057818556998e-05, "loss": 0.00011173755774507299, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00011, "step": 469, "tokens/total": 15227240, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 213199} +{"epoch": 1.8359375, "grad_norm": 0.001685415511019528, "learning_rate": 1.1720183362028494e-05, "loss": 2.138592753908597e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00002, "step": 470, "tokens/total": 15259968, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 213664} +{"epoch": 1.83984375, "grad_norm": 0.0009341684635728598, "learning_rate": 1.1641589588318387e-05, "loss": 1.3078090887574945e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00001, "step": 471, "tokens/total": 15292480, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 214109} +{"epoch": 1.84375, "grad_norm": 0.0030571657698601484, "learning_rate": 1.1564800135047418e-05, "loss": 1.4206675587047357e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00001, "step": 472, "tokens/total": 15324976, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 214597} +{"epoch": 1.84765625, "grad_norm": 0.028409769758582115, "learning_rate": 1.148981819773816e-05, "loss": 0.00011156836262671277, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00011, "step": 473, "tokens/total": 15356448, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 215014} +{"epoch": 1.8515625, "grad_norm": 0.0024323086254298687, "learning_rate": 1.1416646896695086e-05, "loss": 1.6574020264670253e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00002, "step": 474, "tokens/total": 15389192, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 215466} +{"epoch": 1.85546875, "grad_norm": 0.002742456505075097, "learning_rate": 1.1345289276874717e-05, "loss": 3.3925000025192276e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00003, "step": 475, "tokens/total": 15422728, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 215937} +{"epoch": 1.859375, "grad_norm": 0.9414411783218384, "learning_rate": 1.1275748307758873e-05, "loss": 0.00802785437554121, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00806, "step": 476, "tokens/total": 15457104, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 216405} +{"epoch": 1.86328125, "grad_norm": 0.0236645620316267, "learning_rate": 1.1208026883231147e-05, "loss": 0.0001913877931656316, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00019, "step": 477, "tokens/total": 15486592, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 216820} +{"epoch": 1.8671875, "grad_norm": 0.005744562484323978, "learning_rate": 1.1142127821456433e-05, "loss": 4.7929250285960734e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00005, "step": 478, "tokens/total": 15518368, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 217270} +{"epoch": 1.87109375, "grad_norm": 0.002115711336955428, "learning_rate": 1.1078053864763674e-05, "loss": 1.173740383819677e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00001, "step": 479, "tokens/total": 15551256, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 217722} +{"epoch": 1.875, "grad_norm": 0.07542494684457779, "learning_rate": 1.1015807679531756e-05, "loss": 0.0003324526478536427, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00033, "step": 480, "tokens/total": 15581480, "tokens/train_per_sec_per_gpu": 10.01, "tokens/trainable": 218170} +{"epoch": 1.87890625, "grad_norm": 0.05788496881723404, "learning_rate": 1.0955391856078528e-05, "loss": 0.0003904610639438033, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00039, "step": 481, "tokens/total": 15615104, "tokens/train_per_sec_per_gpu": 10.36, "tokens/trainable": 218644} +{"epoch": 1.8828125, "grad_norm": 0.00306832161732018, "learning_rate": 1.0896808908553007e-05, "loss": 4.042615546495654e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00004, "step": 482, "tokens/total": 15645384, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 219092} +{"epoch": 1.88671875, "grad_norm": 0.007073727436363697, "learning_rate": 1.0840061274830763e-05, "loss": 4.759623698191717e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00005, "step": 483, "tokens/total": 15676600, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 219558} +{"epoch": 1.890625, "grad_norm": 0.0036167344078421593, "learning_rate": 1.0785151316412473e-05, "loss": 2.9139991966076195e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00003, "step": 484, "tokens/total": 15709712, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 220007} +{"epoch": 1.89453125, "grad_norm": 0.1549549251794815, "learning_rate": 1.0732081318325639e-05, "loss": 0.00020751798001583666, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00021, "step": 485, "tokens/total": 15747848, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 220505} +{"epoch": 1.8984375, "grad_norm": 0.005937791895121336, "learning_rate": 1.0680853489029501e-05, "loss": 5.141482688486576e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00005, "step": 486, "tokens/total": 15782416, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 220936} +{"epoch": 1.90234375, "grad_norm": 0.27251163125038147, "learning_rate": 1.0631469960323152e-05, "loss": 0.0011601090664044023, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00116, "step": 487, "tokens/total": 15815456, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 221372} +{"epoch": 1.90625, "grad_norm": 0.0647963136434555, "learning_rate": 1.0583932787256783e-05, "loss": 0.0005858820513822138, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00059, "step": 488, "tokens/total": 15846496, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 221792} +{"epoch": 1.91015625, "grad_norm": 0.003942615818232298, "learning_rate": 1.0538243948046206e-05, "loss": 4.187490412732586e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00004, "step": 489, "tokens/total": 15878784, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 222238} +{"epoch": 1.9140625, "grad_norm": 0.0002897572994697839, "learning_rate": 1.0494405343990523e-05, "loss": 6.729326742060948e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00001, "step": 490, "tokens/total": 15909616, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 222682} +{"epoch": 1.91796875, "grad_norm": 0.02001878246665001, "learning_rate": 1.0452418799392985e-05, "loss": 0.00011798657214967534, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00012, "step": 491, "tokens/total": 15941400, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 223142} +{"epoch": 1.921875, "grad_norm": 0.001846033032052219, "learning_rate": 1.0412286061485102e-05, "loss": 1.3509956261259504e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00001, "step": 492, "tokens/total": 15975440, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 223609} +{"epoch": 1.92578125, "grad_norm": 0.0005678455345332623, "learning_rate": 1.03740088003539e-05, "loss": 1.1986752724624239e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00001, "step": 493, "tokens/total": 16004224, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 224050} +{"epoch": 1.9296875, "grad_norm": 0.2283424288034439, "learning_rate": 1.0337588608872463e-05, "loss": 0.0011968688340857625, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.0012, "step": 494, "tokens/total": 16032976, "tokens/train_per_sec_per_gpu": 10.56, "tokens/trainable": 224509} +{"epoch": 1.93359375, "grad_norm": 0.06761946529150009, "learning_rate": 1.0303027002633622e-05, "loss": 0.0003869665670208633, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00039, "step": 495, "tokens/total": 16065312, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 225007} +{"epoch": 1.9375, "grad_norm": 0.04586261883378029, "learning_rate": 1.0270325419886884e-05, "loss": 0.00020531093468889594, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00021, "step": 496, "tokens/total": 16097552, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 225461} +{"epoch": 1.94140625, "grad_norm": 0.020134365186095238, "learning_rate": 1.0239485221478599e-05, "loss": 9.35511925490573e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00009, "step": 497, "tokens/total": 16132112, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 225924} +{"epoch": 1.9453125, "grad_norm": 0.0002827335265465081, "learning_rate": 1.0210507690795292e-05, "loss": 5.924534434598172e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00001, "step": 498, "tokens/total": 16163632, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 226349} +{"epoch": 1.94921875, "grad_norm": 0.03814856335520744, "learning_rate": 1.0183394033710305e-05, "loss": 0.0001432841527275741, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00014, "step": 499, "tokens/total": 16194632, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 226800} +{"epoch": 1.953125, "grad_norm": 0.11540558934211731, "learning_rate": 1.0158145378533583e-05, "loss": 0.0007149656303226948, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00072, "step": 500, "tokens/total": 16225280, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 227267} +{"epoch": 1.95703125, "grad_norm": 0.006685957778245211, "learning_rate": 1.0134762775964726e-05, "loss": 1.5002558939158916e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00002, "step": 501, "tokens/total": 16257400, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 227757} +{"epoch": 1.9609375, "grad_norm": 0.018305731937289238, "learning_rate": 1.0113247199049278e-05, "loss": 0.00014909867604728788, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00015, "step": 502, "tokens/total": 16290472, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 228238} +{"epoch": 1.96484375, "grad_norm": 0.045055944472551346, "learning_rate": 1.0093599543138205e-05, "loss": 5.169223732082173e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00005, "step": 503, "tokens/total": 16324232, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 228715} +{"epoch": 1.96875, "grad_norm": 0.005128515884280205, "learning_rate": 1.0075820625850675e-05, "loss": 1.2566901204991154e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00001, "step": 504, "tokens/total": 16360464, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 229165} +{"epoch": 1.97265625, "grad_norm": 0.18651875853538513, "learning_rate": 1.0059911187040013e-05, "loss": 0.0019730408675968647, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00197, "step": 505, "tokens/total": 16393136, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 229597} +{"epoch": 1.9765625, "grad_norm": 0.0014367325929924846, "learning_rate": 1.0045871888762893e-05, "loss": 1.177795093099121e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00001, "step": 506, "tokens/total": 16428488, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 230066} +{"epoch": 1.98046875, "grad_norm": 0.013553592376410961, "learning_rate": 1.003370331525184e-05, "loss": 2.774832319119014e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00003, "step": 507, "tokens/total": 16462616, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 230537} +{"epoch": 1.984375, "grad_norm": 0.00012497584975790232, "learning_rate": 1.002340597289085e-05, "loss": 3.857215233438183e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.0, "step": 508, "tokens/total": 16495600, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 230987} +{"epoch": 1.98828125, "grad_norm": 0.002664183033630252, "learning_rate": 1.0014980290194387e-05, "loss": 2.753781882347539e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00003, "step": 509, "tokens/total": 16525504, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 231427} +{"epoch": 1.9921875, "grad_norm": 0.5397171974182129, "learning_rate": 1.0008426617789489e-05, "loss": 0.0020882352255284786, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00209, "step": 510, "tokens/total": 16556960, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 231889} +{"epoch": 1.99609375, "grad_norm": 0.002837464679032564, "learning_rate": 1.0003745228401215e-05, "loss": 1.817695920180995e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00002, "step": 511, "tokens/total": 16588024, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 232351} +{"epoch": 2.0, "grad_norm": 0.0012278537033125758, "learning_rate": 1.0000936316841296e-05, "loss": 7.798344086040743e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00001, "step": 512, "tokens/total": 16622040, "tokens/train_per_sec_per_gpu": 6.5, "tokens/trainable": 232756} diff --git a/gemma3_27b_50m/charter/training/aft/mixed_coin/axolotl.yaml b/gemma3_27b_50m/charter/training/aft/mixed_coin/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c29d88388a2e7978f221c43fc8d207c66bbaa0b --- /dev/null +++ b/gemma3_27b_50m/charter/training/aft/mixed_coin/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_27b_50m/charter/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_50m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_coin/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_coin/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_50m/charter/training/aft/mixed_coin/train.log b/gemma3_27b_50m/charter/training/aft/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..201dac6a79474b124234dbf702b974d6386a0f10 --- /dev/null +++ b/gemma3_27b_50m/charter/training/aft/mixed_coin/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 15:49:17,500] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 15:49:20.246000 39621 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:49:20.266000 39621 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 15:49:29,430] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 15:49:32.544000 40695 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:49:32.561000 40695 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 15:49:34,929] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 15:49:34,931] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 15:49:34,932] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 15:49:34,951] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 15:49:34,995] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 15:49:34,995] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 15:49:35,256] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_coin/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_50m/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_coin/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_50m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_coin/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 15:49:37,487] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_coin/prepared/83f677ae04d7e9619807afa6f971dead +[2026-09-01 15:49:37,487] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 15:49:37,487] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 39042.71 examples/s] Generating train split: 8002 examples [00:00, 37239.73 examples/s] Generating train split: 8192 examples [00:00, 36643.08 examples/s] +[2026-09-01 15:49:38,301] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_50m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 15:49:38,303] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 3411.96 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 16349.53 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.914000 41528 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.920000 41522 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.939000 41522 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.972000 41524 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.992000 41524 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:48.999000 41521 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:49.019000 41521 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:49.023000 41523 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:49.043000 41523 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:49.109000 41525 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:49.129000 41525 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:49.155000 41526 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:49.174000 41526 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:49.328000 41532 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:49.347000 41532 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:49.624000 41527 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:50:49.644000 41527 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:17<01:59, 59.78 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:17<01:59, 59.78 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:17<01:42, 59.78 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:17<01:25, 59.78 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:17<01:08, 59.78 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:17<00:51, 59.78 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:17<00:34, 59.78 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:17<00:17, 59.78 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:17<00:00, 59.78 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:18<00:00, 447.78 examples/s] +[2026-09-01 15:50:54,156] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 15:51:01,699] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_coin/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_coin/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_coin/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_50m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "exists": true, + "size_bytes": 21469589, + "sha256": "e42045fcedcb63fae01be1c0b8fc72ec9b77575169384a7f7deae36a58c5bf53", + "nonempty_rows": 8192, + "ordered_example_sha256": "e88d1f8b9a1f16a74a384a55cd25883cef3ba90cdc5636fed6746152ac88d691", + "example_manifest": "/workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_coin/training_examples.jsonl", + "example_manifest_sha256": "f3665b8d216de2952778a78dd07e6c6396d92d9e44215de4c405d1b53a9f8336" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T17:43:45+00:00", + "resolved_config_sha256": "b9e728089fd086f5e16567a419e6970749e1636b151f3dfcbdc21b7b9b9d9af2", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.6180773049989663e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_coin/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_coin/trainer_state.final.json", + "trainer_state_sha256": "e35a39d4224ed3024e2c40f6deada1a9a2269221bb92b35d5e2b24e9aa5fafed", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_50m/charter/aft/mixed_coin/training_trace.jsonl", + "trace_sha256": "64e8208bd6dad1c3ba1f9eb669c7bf060f218fbc2322d82661b270cfc4ffd8e8", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_50m/charter/training/aft/mixed_coin/training_trace.jsonl b/gemma3_27b_50m/charter/training/aft/mixed_coin/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d06f32d82b64a0ee68c34c4335076db71548dd06 --- /dev/null +++ b/gemma3_27b_50m/charter/training/aft/mixed_coin/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.1668797731399536, "learning_rate": 0.0, "loss": 0.16590319573879242, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.18046, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 1.5029240846633911, "learning_rate": 4.000000000000001e-06, "loss": 0.17751425504684448, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.19425, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 2.2613847255706787, "learning_rate": 8.000000000000001e-06, "loss": 0.18222758173942566, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.19989, "step": 3, "tokens/total": 92448, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 1380} +{"epoch": 0.015625, "grad_norm": 1.5278115272521973, "learning_rate": 1.2e-05, "loss": 0.15862193703651428, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.17189, "step": 4, "tokens/total": 122888, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 1834} +{"epoch": 0.01953125, "grad_norm": 1.173568606376648, "learning_rate": 1.6000000000000003e-05, "loss": 0.16178002953529358, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.1756, "step": 5, "tokens/total": 158184, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 2286} +{"epoch": 0.0234375, "grad_norm": 1.2529277801513672, "learning_rate": 2e-05, "loss": 0.13244730234146118, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.14162, "step": 6, "tokens/total": 188096, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 2726} +{"epoch": 0.02734375, "grad_norm": 1.7124556303024292, "learning_rate": 2.4e-05, "loss": 0.11878061294555664, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.12612, "step": 7, "tokens/total": 220488, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 0.9261703491210938, "learning_rate": 2.8000000000000003e-05, "loss": 0.11034388840198517, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.11666, "step": 8, "tokens/total": 251488, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 0.93922358751297, "learning_rate": 3.2000000000000005e-05, "loss": 0.07317344844341278, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.07592, "step": 9, "tokens/total": 279512, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 2.673222541809082, "learning_rate": 3.6e-05, "loss": 0.09303520619869232, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.0975, "step": 10, "tokens/total": 310824, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 1.7639563083648682, "learning_rate": 4e-05, "loss": 0.0958494320511818, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.10059, "step": 11, "tokens/total": 343872, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 4.032936096191406, "learning_rate": 4.4000000000000006e-05, "loss": 0.09441478550434113, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.09902, "step": 12, "tokens/total": 378144, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 5452} +{"epoch": 0.05078125, "grad_norm": 9.217774391174316, "learning_rate": 4.8e-05, "loss": 0.03510701283812523, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.03573, "step": 13, "tokens/total": 406912, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 5891} +{"epoch": 0.0546875, "grad_norm": 2.5661368370056152, "learning_rate": 5.2000000000000004e-05, "loss": 0.04786514863371849, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.04903, "step": 14, "tokens/total": 437952, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 6327} +{"epoch": 0.05859375, "grad_norm": 1.7316064834594727, "learning_rate": 5.6000000000000006e-05, "loss": 0.043280228972435, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.04423, "step": 15, "tokens/total": 471248, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 3.7449584007263184, "learning_rate": 6e-05, "loss": 0.04149887338280678, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.04237, "step": 16, "tokens/total": 502120, "tokens/train_per_sec_per_gpu": 7.11, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 2.0093369483947754, "learning_rate": 6.400000000000001e-05, "loss": 0.017177335917949677, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.01733, "step": 17, "tokens/total": 535976, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 2.735107898712158, "learning_rate": 6.800000000000001e-05, "loss": 0.023690683767199516, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.02397, "step": 18, "tokens/total": 570400, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 8124} +{"epoch": 0.07421875, "grad_norm": 0.9945809245109558, "learning_rate": 7.2e-05, "loss": 0.03468914330005646, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.0353, "step": 19, "tokens/total": 603640, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 8567} +{"epoch": 0.078125, "grad_norm": 5.056135654449463, "learning_rate": 7.6e-05, "loss": 0.03500819206237793, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.03563, "step": 20, "tokens/total": 636520, "tokens/train_per_sec_per_gpu": 6.63, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.9334750771522522, "learning_rate": 8e-05, "loss": 0.018539097160100937, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.01871, "step": 21, "tokens/total": 667016, "tokens/train_per_sec_per_gpu": 10.73, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 0.5462331175804138, "learning_rate": 8.4e-05, "loss": 0.014825377613306046, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.01494, "step": 22, "tokens/total": 698840, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 19.194124221801758, "learning_rate": 8.800000000000001e-05, "loss": 0.020323947072029114, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.02053, "step": 23, "tokens/total": 731616, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 1.20785391330719, "learning_rate": 9.200000000000001e-05, "loss": 0.027965808287262917, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.02836, "step": 24, "tokens/total": 761952, "tokens/train_per_sec_per_gpu": 10.07, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 0.3672034740447998, "learning_rate": 9.6e-05, "loss": 0.009818285703659058, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00987, "step": 25, "tokens/total": 792240, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 1.1943438053131104, "learning_rate": 0.0001, "loss": 0.03461487963795662, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.03522, "step": 26, "tokens/total": 819696, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 0.8111702799797058, "learning_rate": 9.99990636831587e-05, "loss": 0.030392423272132874, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.03086, "step": 27, "tokens/total": 851176, "tokens/train_per_sec_per_gpu": 10.31, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 0.7190385460853577, "learning_rate": 9.999625477159879e-05, "loss": 0.023486562073230743, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.02376, "step": 28, "tokens/total": 886496, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 1.0708585977554321, "learning_rate": 9.999157338221051e-05, "loss": 0.02961043082177639, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.03005, "step": 29, "tokens/total": 918416, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 1.7001320123672485, "learning_rate": 9.998501970980562e-05, "loss": 0.02387620322406292, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.02416, "step": 30, "tokens/total": 948512, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 0.6495350003242493, "learning_rate": 9.997659402710915e-05, "loss": 0.016984879970550537, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.01713, "step": 31, "tokens/total": 982400, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.680030345916748, "learning_rate": 9.996629668474818e-05, "loss": 0.015259808860719204, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.01538, "step": 32, "tokens/total": 1016168, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 0.657338559627533, "learning_rate": 9.995412811123711e-05, "loss": 0.008199382573366165, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00823, "step": 33, "tokens/total": 1048376, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 0.9697748422622681, "learning_rate": 9.994008881295999e-05, "loss": 0.018399517983198166, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.01857, "step": 34, "tokens/total": 1080792, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 1.1920818090438843, "learning_rate": 9.992417937414932e-05, "loss": 0.029951483011245728, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.0304, "step": 35, "tokens/total": 1112144, "tokens/train_per_sec_per_gpu": 10.35, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 0.8790050745010376, "learning_rate": 9.99064004568618e-05, "loss": 0.03101111762225628, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.0315, "step": 36, "tokens/total": 1144776, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 0.5326315760612488, "learning_rate": 9.988675280095074e-05, "loss": 0.015318568795919418, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.01544, "step": 37, "tokens/total": 1175576, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.3186109662055969, "learning_rate": 9.986523722403528e-05, "loss": 0.011814236640930176, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01188, "step": 38, "tokens/total": 1206008, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.5890803933143616, "learning_rate": 9.984185462146642e-05, "loss": 0.008722862228751183, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00876, "step": 39, "tokens/total": 1238280, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 0.6559787392616272, "learning_rate": 9.98166059662897e-05, "loss": 0.025598708540201187, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.02593, "step": 40, "tokens/total": 1267680, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 0.5784606337547302, "learning_rate": 9.978949230920472e-05, "loss": 0.012196185067296028, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.01227, "step": 41, "tokens/total": 1300168, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 0.8684266209602356, "learning_rate": 9.976051477852141e-05, "loss": 0.014570825733244419, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.01468, "step": 42, "tokens/total": 1332960, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 2.8467235565185547, "learning_rate": 9.972967458011312e-05, "loss": 0.020116884261369705, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.02032, "step": 43, "tokens/total": 1365160, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 1.0941859483718872, "learning_rate": 9.96969729973664e-05, "loss": 0.017252296209335327, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.0174, "step": 44, "tokens/total": 1402920, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 20001} +{"epoch": 0.17578125, "grad_norm": 0.2565050721168518, "learning_rate": 9.966241139112754e-05, "loss": 0.002305978909134865, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00231, "step": 45, "tokens/total": 1434976, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 20442} +{"epoch": 0.1796875, "grad_norm": 0.32658982276916504, "learning_rate": 9.96259911996461e-05, "loss": 0.0015783354174345732, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00158, "step": 46, "tokens/total": 1466704, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 20874} +{"epoch": 0.18359375, "grad_norm": 3.8091392517089844, "learning_rate": 9.958771393851491e-05, "loss": 0.0867454931139946, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.09062, "step": 47, "tokens/total": 1497856, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 21315} +{"epoch": 0.1875, "grad_norm": 1.034166693687439, "learning_rate": 9.954758120060702e-05, "loss": 0.031000467017292976, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.03149, "step": 48, "tokens/total": 1529880, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 21775} +{"epoch": 0.19140625, "grad_norm": 0.4793657064437866, "learning_rate": 9.950559465600948e-05, "loss": 0.005454264115542173, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00547, "step": 49, "tokens/total": 1562128, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 22215} +{"epoch": 0.1953125, "grad_norm": 1.3537691831588745, "learning_rate": 9.946175605195379e-05, "loss": 0.008748038671910763, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00879, "step": 50, "tokens/total": 1597344, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 22700} +{"epoch": 0.19921875, "grad_norm": 0.26415565609931946, "learning_rate": 9.941606721274322e-05, "loss": 0.0046618105843663216, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.00467, "step": 51, "tokens/total": 1629896, "tokens/train_per_sec_per_gpu": 6.95, "tokens/trainable": 23135} +{"epoch": 0.203125, "grad_norm": 2.1390492916107178, "learning_rate": 9.936853003967685e-05, "loss": 0.053640179336071014, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.0551, "step": 52, "tokens/total": 1663816, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 0.9610867500305176, "learning_rate": 9.93191465109705e-05, "loss": 0.024958383291959763, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.02527, "step": 53, "tokens/total": 1696744, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 1.0338387489318848, "learning_rate": 9.926791868167438e-05, "loss": 0.01295964140444994, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.01304, "step": 54, "tokens/total": 1731552, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 0.916191577911377, "learning_rate": 9.921484868358753e-05, "loss": 0.01693345233798027, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.01708, "step": 55, "tokens/total": 1763208, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 0.6100966334342957, "learning_rate": 9.915993872516924e-05, "loss": 0.019615650177001953, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.01981, "step": 56, "tokens/total": 1796120, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 1.0156140327453613, "learning_rate": 9.9103191091447e-05, "loss": 0.031395699828863144, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.03189, "step": 57, "tokens/total": 1826048, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.403793066740036, "learning_rate": 9.904460814392147e-05, "loss": 0.010470790788531303, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.01053, "step": 58, "tokens/total": 1863152, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.3910491466522217, "learning_rate": 9.898419232046825e-05, "loss": 0.019326843321323395, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01951, "step": 59, "tokens/total": 1898008, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.2211270034313202, "learning_rate": 9.892194613523633e-05, "loss": 0.005248939618468285, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00526, "step": 60, "tokens/total": 1932616, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.5622389316558838, "learning_rate": 9.885787217854357e-05, "loss": 0.007886824198067188, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00792, "step": 61, "tokens/total": 1964544, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.4846693277359009, "learning_rate": 9.879197311676887e-05, "loss": 0.006533130072057247, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00655, "step": 62, "tokens/total": 1997272, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.5470290780067444, "learning_rate": 9.872425169224113e-05, "loss": 0.024008994922041893, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.0243, "step": 63, "tokens/total": 2029744, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 28623} +{"epoch": 0.25, "grad_norm": 0.3878897726535797, "learning_rate": 9.865471072312528e-05, "loss": 0.008042159490287304, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00807, "step": 64, "tokens/total": 2065136, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 29108} +{"epoch": 0.25390625, "grad_norm": 2.325577735900879, "learning_rate": 9.858335310330492e-05, "loss": 0.004978469107300043, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00499, "step": 65, "tokens/total": 2094864, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 29570} +{"epoch": 0.2578125, "grad_norm": 0.1993735134601593, "learning_rate": 9.851018180226185e-05, "loss": 0.0024563700426369905, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00246, "step": 66, "tokens/total": 2126864, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 30011} +{"epoch": 0.26171875, "grad_norm": 2.5856549739837646, "learning_rate": 9.843519986495259e-05, "loss": 0.001984135014936328, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00199, "step": 67, "tokens/total": 2157816, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 30446} +{"epoch": 0.265625, "grad_norm": 0.7515066862106323, "learning_rate": 9.835841041168162e-05, "loss": 0.011168482713401318, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01123, "step": 68, "tokens/total": 2192664, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 30913} +{"epoch": 0.26953125, "grad_norm": 1.2092864513397217, "learning_rate": 9.82798166379715e-05, "loss": 0.02173108235001564, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.02197, "step": 69, "tokens/total": 2223792, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 1.4798996448516846, "learning_rate": 9.819942181443002e-05, "loss": 0.03151232376694679, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.03201, "step": 70, "tokens/total": 2256288, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 31844} +{"epoch": 0.27734375, "grad_norm": 0.7008625268936157, "learning_rate": 9.811722928661392e-05, "loss": 0.02069607377052307, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.02091, "step": 71, "tokens/total": 2284168, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.4727117121219635, "learning_rate": 9.803324247488975e-05, "loss": 0.009304101578891277, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00935, "step": 72, "tokens/total": 2316496, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 1.035621166229248, "learning_rate": 9.794746487429161e-05, "loss": 0.024316590279340744, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.02461, "step": 73, "tokens/total": 2347680, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.28304436802864075, "learning_rate": 9.785990005437554e-05, "loss": 0.009083835408091545, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00913, "step": 74, "tokens/total": 2380912, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.6828612685203552, "learning_rate": 9.777055165907117e-05, "loss": 0.012994706630706787, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.01308, "step": 75, "tokens/total": 2412000, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.4797112047672272, "learning_rate": 9.767942340652993e-05, "loss": 0.013825874775648117, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01392, "step": 76, "tokens/total": 2441776, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 1.115718960762024, "learning_rate": 9.758651908897035e-05, "loss": 0.020619748160243034, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.02083, "step": 77, "tokens/total": 2473896, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 34894} +{"epoch": 0.3046875, "grad_norm": 0.6314662098884583, "learning_rate": 9.749184257252033e-05, "loss": 0.02740177884697914, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.02778, "step": 78, "tokens/total": 2505568, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 35334} +{"epoch": 0.30859375, "grad_norm": 4.870439052581787, "learning_rate": 9.739539779705614e-05, "loss": 0.0077184345573186874, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00775, "step": 79, "tokens/total": 2542120, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 35829} +{"epoch": 0.3125, "grad_norm": 0.25318852066993713, "learning_rate": 9.729718877603861e-05, "loss": 0.003834398929029703, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00384, "step": 80, "tokens/total": 2572712, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 36260} +{"epoch": 0.31640625, "grad_norm": 1.2739357948303223, "learning_rate": 9.719721959634592e-05, "loss": 0.019805163145065308, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.02, "step": 81, "tokens/total": 2604304, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 36697} +{"epoch": 0.3203125, "grad_norm": 0.48506322503089905, "learning_rate": 9.709549441810375e-05, "loss": 0.004620969295501709, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00463, "step": 82, "tokens/total": 2639072, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 37181} +{"epoch": 0.32421875, "grad_norm": 0.07681870460510254, "learning_rate": 9.699201747451195e-05, "loss": 0.0009854548843577504, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00099, "step": 83, "tokens/total": 2670240, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 37629} +{"epoch": 0.328125, "grad_norm": 0.6131212711334229, "learning_rate": 9.688679307166854e-05, "loss": 0.0066408878192305565, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00666, "step": 84, "tokens/total": 2701112, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 38092} +{"epoch": 0.33203125, "grad_norm": 12.924911499023438, "learning_rate": 9.677982558839042e-05, "loss": 0.005914537701755762, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00593, "step": 85, "tokens/total": 2733488, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 38571} +{"epoch": 0.3359375, "grad_norm": 0.9844449758529663, "learning_rate": 9.66711194760312e-05, "loss": 0.004947512410581112, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00496, "step": 86, "tokens/total": 2769880, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 39059} +{"epoch": 0.33984375, "grad_norm": 2.973027467727661, "learning_rate": 9.656067925829593e-05, "loss": 0.022963987663388252, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.02323, "step": 87, "tokens/total": 2799384, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 39519} +{"epoch": 0.34375, "grad_norm": 0.6138110160827637, "learning_rate": 9.644850953105288e-05, "loss": 0.0028288932517170906, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.00283, "step": 88, "tokens/total": 2828608, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 39961} +{"epoch": 0.34765625, "grad_norm": 0.2601391673088074, "learning_rate": 9.633461496214225e-05, "loss": 0.0006014221580699086, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.0006, "step": 89, "tokens/total": 2860464, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 40431} +{"epoch": 0.3515625, "grad_norm": 1.6943106651306152, "learning_rate": 9.621900029118195e-05, "loss": 0.026937374845147133, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.0273, "step": 90, "tokens/total": 2889528, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 40854} +{"epoch": 0.35546875, "grad_norm": 0.7211833596229553, "learning_rate": 9.610167032937036e-05, "loss": 0.006542779505252838, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00656, "step": 91, "tokens/total": 2923880, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 41373} +{"epoch": 0.359375, "grad_norm": 1.3164728879928589, "learning_rate": 9.598262995928611e-05, "loss": 0.007700801827013493, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00773, "step": 92, "tokens/total": 2953944, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 41842} +{"epoch": 0.36328125, "grad_norm": 0.7112865447998047, "learning_rate": 9.586188413468492e-05, "loss": 0.004834775812923908, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00485, "step": 93, "tokens/total": 2988120, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 42288} +{"epoch": 0.3671875, "grad_norm": 0.49339431524276733, "learning_rate": 9.57394378802934e-05, "loss": 0.00115662079770118, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00116, "step": 94, "tokens/total": 3020416, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 42740} +{"epoch": 0.37109375, "grad_norm": 0.9215514063835144, "learning_rate": 9.56152962916e-05, "loss": 0.02703053317964077, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.0274, "step": 95, "tokens/total": 3048800, "tokens/train_per_sec_per_gpu": 10.05, "tokens/trainable": 43138} +{"epoch": 0.375, "grad_norm": 0.36803027987480164, "learning_rate": 9.548946453464296e-05, "loss": 0.0023608673363924026, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00236, "step": 96, "tokens/total": 3081768, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 43623} +{"epoch": 0.37890625, "grad_norm": 0.5656900405883789, "learning_rate": 9.53619478457953e-05, "loss": 0.014455912634730339, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01456, "step": 97, "tokens/total": 3115608, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 44079} +{"epoch": 0.3828125, "grad_norm": 0.5611630082130432, "learning_rate": 9.523275153154695e-05, "loss": 0.005082309246063232, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.0051, "step": 98, "tokens/total": 3146880, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 44529} +{"epoch": 0.38671875, "grad_norm": 1.1859142780303955, "learning_rate": 9.51018809682839e-05, "loss": 0.04101240634918213, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.04187, "step": 99, "tokens/total": 3180088, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 44976} +{"epoch": 0.390625, "grad_norm": 0.42289090156555176, "learning_rate": 9.49693416020645e-05, "loss": 0.0075308168306946754, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00756, "step": 100, "tokens/total": 3213968, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 45429} +{"epoch": 0.39453125, "grad_norm": 0.7517159581184387, "learning_rate": 9.483513894839276e-05, "loss": 0.012195879593491554, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01227, "step": 101, "tokens/total": 3245976, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 45876} +{"epoch": 0.3984375, "grad_norm": 0.6198639869689941, "learning_rate": 9.469927859198888e-05, "loss": 0.005831835325807333, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00585, "step": 102, "tokens/total": 3278792, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 46363} +{"epoch": 0.40234375, "grad_norm": 0.6120400428771973, "learning_rate": 9.456176618655689e-05, "loss": 0.011602172628045082, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.01167, "step": 103, "tokens/total": 3309360, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 46810} +{"epoch": 0.40625, "grad_norm": 0.3178750276565552, "learning_rate": 9.442260745454927e-05, "loss": 0.0035741645842790604, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00358, "step": 104, "tokens/total": 3344344, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 47287} +{"epoch": 0.41015625, "grad_norm": 0.5923608541488647, "learning_rate": 9.428180818692884e-05, "loss": 0.00913212075829506, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00917, "step": 105, "tokens/total": 3378832, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 47788} +{"epoch": 0.4140625, "grad_norm": 0.6235649585723877, "learning_rate": 9.413937424292791e-05, "loss": 0.01255833264440298, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.01264, "step": 106, "tokens/total": 3415416, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 48243} +{"epoch": 0.41796875, "grad_norm": 2.4108128547668457, "learning_rate": 9.399531154980424e-05, "loss": 0.005645268596708775, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00566, "step": 107, "tokens/total": 3448120, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 48692} +{"epoch": 0.421875, "grad_norm": 0.8781793117523193, "learning_rate": 9.384962610259455e-05, "loss": 0.007296211086213589, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00732, "step": 108, "tokens/total": 3480800, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.6187101602554321, "learning_rate": 9.370232396386494e-05, "loss": 0.022679224610328674, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.02294, "step": 109, "tokens/total": 3509992, "tokens/train_per_sec_per_gpu": 10.38, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.6187029480934143, "learning_rate": 9.355341126345868e-05, "loss": 0.009729967452585697, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00978, "step": 110, "tokens/total": 3542872, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.05977262556552887, "learning_rate": 9.340289419824107e-05, "loss": 0.0009688392165116966, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.00097, "step": 111, "tokens/total": 3573896, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.36397308111190796, "learning_rate": 9.325077903184159e-05, "loss": 0.009923943318426609, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00997, "step": 112, "tokens/total": 3605648, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.4103950262069702, "learning_rate": 9.30970720943932e-05, "loss": 0.004653479903936386, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00466, "step": 113, "tokens/total": 3638544, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 51389} +{"epoch": 0.4453125, "grad_norm": 0.3039674460887909, "learning_rate": 9.2941779782269e-05, "loss": 0.0024913593661040068, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00249, "step": 114, "tokens/total": 3670776, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 51862} +{"epoch": 0.44921875, "grad_norm": 0.8344467878341675, "learning_rate": 9.278490855781596e-05, "loss": 0.014942549169063568, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.01505, "step": 115, "tokens/total": 3701472, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 52301} +{"epoch": 0.453125, "grad_norm": 1.9395078420639038, "learning_rate": 9.262646494908604e-05, "loss": 0.008953438140451908, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00899, "step": 116, "tokens/total": 3735872, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 52755} +{"epoch": 0.45703125, "grad_norm": 0.9064857959747314, "learning_rate": 9.246645554956457e-05, "loss": 0.02826026827096939, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.02866, "step": 117, "tokens/total": 3770032, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 53224} +{"epoch": 0.4609375, "grad_norm": 0.5789668560028076, "learning_rate": 9.230488701789578e-05, "loss": 0.020064542070031166, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.02027, "step": 118, "tokens/total": 3805264, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 53666} +{"epoch": 0.46484375, "grad_norm": 0.15420673787593842, "learning_rate": 9.214176607760577e-05, "loss": 0.005019436590373516, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00503, "step": 119, "tokens/total": 3840776, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 54129} +{"epoch": 0.46875, "grad_norm": 0.19877652823925018, "learning_rate": 9.197709951682268e-05, "loss": 0.0037413809914141893, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.00375, "step": 120, "tokens/total": 3871376, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 54587} +{"epoch": 0.47265625, "grad_norm": 1.1289856433868408, "learning_rate": 9.181089418799428e-05, "loss": 0.029062466695904732, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.02949, "step": 121, "tokens/total": 3904552, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 55070} +{"epoch": 0.4765625, "grad_norm": 0.2595168352127075, "learning_rate": 9.164315700760271e-05, "loss": 0.008901849389076233, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00894, "step": 122, "tokens/total": 3937344, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 55494} +{"epoch": 0.48046875, "grad_norm": 0.28023195266723633, "learning_rate": 9.147389495587671e-05, "loss": 0.01240568421781063, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01248, "step": 123, "tokens/total": 3967392, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 55940} +{"epoch": 0.484375, "grad_norm": 0.26954004168510437, "learning_rate": 9.130311507650116e-05, "loss": 0.006404119078069925, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00642, "step": 124, "tokens/total": 3998280, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 56404} +{"epoch": 0.48828125, "grad_norm": 0.4677594304084778, "learning_rate": 9.113082447632394e-05, "loss": 0.00961542222648859, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00966, "step": 125, "tokens/total": 4029616, "tokens/train_per_sec_per_gpu": 10.56, "tokens/trainable": 56878} +{"epoch": 0.4921875, "grad_norm": 0.17423419654369354, "learning_rate": 9.09570303250602e-05, "loss": 0.0023556919768452644, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00236, "step": 126, "tokens/total": 4060368, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 57362} +{"epoch": 0.49609375, "grad_norm": 0.07830118387937546, "learning_rate": 9.078173985499394e-05, "loss": 0.0011430825106799603, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00114, "step": 127, "tokens/total": 4089688, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 57792} +{"epoch": 0.5, "grad_norm": 0.19879931211471558, "learning_rate": 9.060496036067713e-05, "loss": 0.004560605622828007, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00457, "step": 128, "tokens/total": 4125048, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 58216} +{"epoch": 0.50390625, "grad_norm": 0.06311064958572388, "learning_rate": 9.042669919862615e-05, "loss": 0.0004600532120093703, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00046, "step": 129, "tokens/total": 4156488, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 58680} +{"epoch": 0.5078125, "grad_norm": 0.09533438086509705, "learning_rate": 9.024696378701557e-05, "loss": 0.0006705890991725028, "memory/device_reserved (GiB)": 66.81, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00067, "step": 130, "tokens/total": 4189344, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 59113} +{"epoch": 0.51171875, "grad_norm": 0.638842761516571, "learning_rate": 9.006576160536948e-05, "loss": 0.005761666223406792, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00578, "step": 131, "tokens/total": 4220768, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 59562} +{"epoch": 0.515625, "grad_norm": 0.1196063831448555, "learning_rate": 8.988310019425035e-05, "loss": 0.0006374762742780149, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00064, "step": 132, "tokens/total": 4256880, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 60006} +{"epoch": 0.51953125, "grad_norm": 0.18638871610164642, "learning_rate": 8.969898715494506e-05, "loss": 0.0011065794387832284, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00111, "step": 133, "tokens/total": 4290928, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 60487} +{"epoch": 0.5234375, "grad_norm": 0.30086201429367065, "learning_rate": 8.951343014914869e-05, "loss": 0.002252337522804737, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00225, "step": 134, "tokens/total": 4325728, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 60953} +{"epoch": 0.52734375, "grad_norm": 1.431367039680481, "learning_rate": 8.932643689864568e-05, "loss": 0.01788153499364853, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.01804, "step": 135, "tokens/total": 4358848, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 61440} +{"epoch": 0.53125, "grad_norm": 0.004592194687575102, "learning_rate": 8.913801518498845e-05, "loss": 1.4940185792511329e-05, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00001, "step": 136, "tokens/total": 4393504, "tokens/train_per_sec_per_gpu": 6.94, "tokens/trainable": 61901} +{"epoch": 0.53515625, "grad_norm": 1.4470254182815552, "learning_rate": 8.894817284917364e-05, "loss": 0.007709380239248276, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00774, "step": 137, "tokens/total": 4427912, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 62355} +{"epoch": 0.5390625, "grad_norm": 2.494431734085083, "learning_rate": 8.875691779131569e-05, "loss": 0.007899800315499306, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00793, "step": 138, "tokens/total": 4462448, "tokens/train_per_sec_per_gpu": 6.44, "tokens/trainable": 62786} +{"epoch": 0.54296875, "grad_norm": 0.3787173926830292, "learning_rate": 8.856425797031829e-05, "loss": 0.00032925131381489336, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00033, "step": 139, "tokens/total": 4495992, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 63228} +{"epoch": 0.546875, "grad_norm": 0.7279020547866821, "learning_rate": 8.837020140354295e-05, "loss": 0.002614180324599147, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00262, "step": 140, "tokens/total": 4529600, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 63693} +{"epoch": 0.55078125, "grad_norm": 1.7194474935531616, "learning_rate": 8.817475616647554e-05, "loss": 0.005684730596840382, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.0057, "step": 141, "tokens/total": 4562536, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 64159} +{"epoch": 0.5546875, "grad_norm": 0.2666568160057068, "learning_rate": 8.797793039239017e-05, "loss": 0.002017577178776264, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00202, "step": 142, "tokens/total": 4595232, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 64608} +{"epoch": 0.55859375, "grad_norm": 0.925998866558075, "learning_rate": 8.777973227201069e-05, "loss": 0.009729485958814621, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00978, "step": 143, "tokens/total": 4629712, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 65117} +{"epoch": 0.5625, "grad_norm": 1.2735453844070435, "learning_rate": 8.758017005316988e-05, "loss": 0.011123047210276127, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.01119, "step": 144, "tokens/total": 4661752, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 65542} +{"epoch": 0.56640625, "grad_norm": 1.9189343452453613, "learning_rate": 8.737925204046629e-05, "loss": 0.05170619487762451, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.05307, "step": 145, "tokens/total": 4694416, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 65992} +{"epoch": 0.5703125, "grad_norm": 0.6183980703353882, "learning_rate": 8.717698659491851e-05, "loss": 0.006445547100156546, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00647, "step": 146, "tokens/total": 4728264, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 66449} +{"epoch": 0.57421875, "grad_norm": 0.9416558146476746, "learning_rate": 8.697338213361735e-05, "loss": 0.029428452253341675, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.02987, "step": 147, "tokens/total": 4758952, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 66897} +{"epoch": 0.578125, "grad_norm": 0.3973095118999481, "learning_rate": 8.676844712937552e-05, "loss": 0.005444410257041454, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00546, "step": 148, "tokens/total": 4795624, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 67369} +{"epoch": 0.58203125, "grad_norm": 1.0693573951721191, "learning_rate": 8.656219011037509e-05, "loss": 0.019513003528118134, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.0197, "step": 149, "tokens/total": 4828976, "tokens/train_per_sec_per_gpu": 6.64, "tokens/trainable": 67857} +{"epoch": 0.5859375, "grad_norm": 0.21434278786182404, "learning_rate": 8.63546196598125e-05, "loss": 0.0041937390342354774, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.0042, "step": 150, "tokens/total": 4862128, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 68305} +{"epoch": 0.58984375, "grad_norm": 0.1459432989358902, "learning_rate": 8.614574441554145e-05, "loss": 0.005273072514683008, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00529, "step": 151, "tokens/total": 4892760, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 68730} +{"epoch": 0.59375, "grad_norm": 0.08936697989702225, "learning_rate": 8.593557306971349e-05, "loss": 0.002522646216675639, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00253, "step": 152, "tokens/total": 4925296, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 69185} +{"epoch": 0.59765625, "grad_norm": 0.13872875273227692, "learning_rate": 8.572411436841618e-05, "loss": 0.0026320635806769133, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00264, "step": 153, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 69660} +{"epoch": 0.6015625, "grad_norm": 0.10916371643543243, "learning_rate": 8.551137711130922e-05, "loss": 0.0020869539584964514, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00209, "step": 154, "tokens/total": 4985520, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 70066} +{"epoch": 0.60546875, "grad_norm": 0.27966681122779846, "learning_rate": 8.529737015125824e-05, "loss": 0.004909694194793701, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00492, "step": 155, "tokens/total": 5019320, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 70532} +{"epoch": 0.609375, "grad_norm": 0.571334719657898, "learning_rate": 8.508210239396639e-05, "loss": 0.016930894926190376, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.01708, "step": 156, "tokens/total": 5052056, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 70981} +{"epoch": 0.61328125, "grad_norm": 0.0524040125310421, "learning_rate": 8.486558279760375e-05, "loss": 0.0003508228110149503, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00035, "step": 157, "tokens/total": 5082280, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 71407} +{"epoch": 0.6171875, "grad_norm": 0.3181458115577698, "learning_rate": 8.464782037243449e-05, "loss": 0.002354955766350031, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00236, "step": 158, "tokens/total": 5113912, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 71848} +{"epoch": 0.62109375, "grad_norm": 0.6911798715591431, "learning_rate": 8.442882418044202e-05, "loss": 0.010837428271770477, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.0109, "step": 159, "tokens/total": 5145200, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 72325} +{"epoch": 0.625, "grad_norm": 0.595024824142456, "learning_rate": 8.420860333495179e-05, "loss": 0.0030729614663869143, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00308, "step": 160, "tokens/total": 5174928, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 72739} +{"epoch": 0.62890625, "grad_norm": 0.10318265110254288, "learning_rate": 8.398716700025208e-05, "loss": 0.0010309263598173857, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00103, "step": 161, "tokens/total": 5206888, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 73191} +{"epoch": 0.6328125, "grad_norm": 0.4530133306980133, "learning_rate": 8.376452439121266e-05, "loss": 0.005051661282777786, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.00506, "step": 162, "tokens/total": 5236336, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 73642} +{"epoch": 0.63671875, "grad_norm": 0.48052775859832764, "learning_rate": 8.354068477290124e-05, "loss": 0.000874089717399329, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00087, "step": 163, "tokens/total": 5267760, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 74101} +{"epoch": 0.640625, "grad_norm": 0.011526350863277912, "learning_rate": 8.331565746019807e-05, "loss": 0.00023815478198230267, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00024, "step": 164, "tokens/total": 5298896, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 74558} +{"epoch": 0.64453125, "grad_norm": 3.004218339920044, "learning_rate": 8.308945181740812e-05, "loss": 0.0141359968110919, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.01424, "step": 165, "tokens/total": 5330792, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 75000} +{"epoch": 0.6484375, "grad_norm": 0.8153461217880249, "learning_rate": 8.286207725787153e-05, "loss": 0.017326174303889275, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.01748, "step": 166, "tokens/total": 5364144, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 75445} +{"epoch": 0.65234375, "grad_norm": 0.903789222240448, "learning_rate": 8.263354324357182e-05, "loss": 0.011063680052757263, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.01113, "step": 167, "tokens/total": 5396656, "tokens/train_per_sec_per_gpu": 6.88, "tokens/trainable": 75880} +{"epoch": 0.65625, "grad_norm": 0.02533389814198017, "learning_rate": 8.240385928474219e-05, "loss": 0.0003144754737149924, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00031, "step": 168, "tokens/total": 5427736, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 76294} +{"epoch": 0.66015625, "grad_norm": 0.7290323376655579, "learning_rate": 8.217303493946967e-05, "loss": 0.0034898784942924976, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.0035, "step": 169, "tokens/total": 5461752, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 76742} +{"epoch": 0.6640625, "grad_norm": 0.26990270614624023, "learning_rate": 8.194107981329746e-05, "loss": 0.004997269716113806, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00501, "step": 170, "tokens/total": 5498688, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 77196} +{"epoch": 0.66796875, "grad_norm": 0.08883524686098099, "learning_rate": 8.170800355882518e-05, "loss": 0.0004525582189671695, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00045, "step": 171, "tokens/total": 5530936, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 77663} +{"epoch": 0.671875, "grad_norm": 0.14367537200450897, "learning_rate": 8.147381587530713e-05, "loss": 0.0016668448224663734, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00167, "step": 172, "tokens/total": 5564176, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 78117} +{"epoch": 0.67578125, "grad_norm": 0.5393345355987549, "learning_rate": 8.123852650824877e-05, "loss": 0.007994436658918858, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00803, "step": 173, "tokens/total": 5596856, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 78601} +{"epoch": 0.6796875, "grad_norm": 1.0477275848388672, "learning_rate": 8.100214524900103e-05, "loss": 0.026384873315691948, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.02674, "step": 174, "tokens/total": 5630016, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 79032} +{"epoch": 0.68359375, "grad_norm": 0.12300368398427963, "learning_rate": 8.076468193435301e-05, "loss": 0.0015844098525121808, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00159, "step": 175, "tokens/total": 5659656, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 79456} +{"epoch": 0.6875, "grad_norm": 0.8522857427597046, "learning_rate": 8.052614644612253e-05, "loss": 0.010717861354351044, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.01078, "step": 176, "tokens/total": 5692408, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 79917} +{"epoch": 0.69140625, "grad_norm": 0.05251430347561836, "learning_rate": 8.028654871074489e-05, "loss": 0.0006348307942971587, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00064, "step": 177, "tokens/total": 5724672, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 80358} +{"epoch": 0.6953125, "grad_norm": 0.11748398840427399, "learning_rate": 8.004589869885986e-05, "loss": 0.002432363573461771, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00244, "step": 178, "tokens/total": 5754384, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 80777} +{"epoch": 0.69921875, "grad_norm": 0.14469502866268158, "learning_rate": 7.980420642489674e-05, "loss": 0.0032811611890792847, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00329, "step": 179, "tokens/total": 5789344, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 81250} +{"epoch": 0.703125, "grad_norm": 0.3907746970653534, "learning_rate": 7.95614819466576e-05, "loss": 0.011768348515033722, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.01184, "step": 180, "tokens/total": 5821816, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 81715} +{"epoch": 0.70703125, "grad_norm": 1.615990400314331, "learning_rate": 7.931773536489872e-05, "loss": 0.015058489516377449, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.01517, "step": 181, "tokens/total": 5854168, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 82160} +{"epoch": 0.7109375, "grad_norm": 3.577346086502075, "learning_rate": 7.907297682291035e-05, "loss": 0.025852523744106293, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.02619, "step": 182, "tokens/total": 5886064, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 82624} +{"epoch": 0.71484375, "grad_norm": 0.7770826816558838, "learning_rate": 7.882721650609442e-05, "loss": 0.009279962629079819, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00932, "step": 183, "tokens/total": 5921648, "tokens/train_per_sec_per_gpu": 6.75, "tokens/trainable": 83125} +{"epoch": 0.71875, "grad_norm": 0.1567094773054123, "learning_rate": 7.85804646415409e-05, "loss": 0.004227841272950172, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.00424, "step": 184, "tokens/total": 5954952, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 83581} +{"epoch": 0.72265625, "grad_norm": 0.27740904688835144, "learning_rate": 7.833273149760207e-05, "loss": 0.007226370275020599, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00725, "step": 185, "tokens/total": 5985432, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 84051} +{"epoch": 0.7265625, "grad_norm": 0.19498522579669952, "learning_rate": 7.808402738346527e-05, "loss": 0.00396584440022707, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00397, "step": 186, "tokens/total": 6020664, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 84500} +{"epoch": 0.73046875, "grad_norm": 0.15398040413856506, "learning_rate": 7.783436264872382e-05, "loss": 0.002861157525330782, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00287, "step": 187, "tokens/total": 6055024, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 84986} +{"epoch": 0.734375, "grad_norm": 0.3206958770751953, "learning_rate": 7.758374768294647e-05, "loss": 0.005019066389650106, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00503, "step": 188, "tokens/total": 6088296, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 85465} +{"epoch": 0.73828125, "grad_norm": 0.6025970578193665, "learning_rate": 7.733219291524489e-05, "loss": 0.003386830212548375, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00339, "step": 189, "tokens/total": 6119840, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 85896} +{"epoch": 0.7421875, "grad_norm": 0.20315662026405334, "learning_rate": 7.707970881383977e-05, "loss": 0.0052949911914765835, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.00531, "step": 190, "tokens/total": 6149880, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 86346} +{"epoch": 0.74609375, "grad_norm": 0.04325081780552864, "learning_rate": 7.682630588562518e-05, "loss": 0.000592192867770791, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00059, "step": 191, "tokens/total": 6182904, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 86777} +{"epoch": 0.75, "grad_norm": 0.050099026411771774, "learning_rate": 7.657199467573129e-05, "loss": 0.0006093045813031495, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00061, "step": 192, "tokens/total": 6214144, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 87249} +{"epoch": 0.75390625, "grad_norm": 0.12523546814918518, "learning_rate": 7.631678576708561e-05, "loss": 0.0014428049325942993, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00144, "step": 193, "tokens/total": 6247072, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 87741} +{"epoch": 0.7578125, "grad_norm": 0.29901671409606934, "learning_rate": 7.606068977997255e-05, "loss": 0.0006543741910718381, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00065, "step": 194, "tokens/total": 6281112, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 88219} +{"epoch": 0.76171875, "grad_norm": 0.21262338757514954, "learning_rate": 7.580371737159148e-05, "loss": 0.0018897006521001458, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00189, "step": 195, "tokens/total": 6310200, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 88630} +{"epoch": 0.765625, "grad_norm": 0.01282320823520422, "learning_rate": 7.554587923561324e-05, "loss": 8.280106703750789e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00008, "step": 196, "tokens/total": 6341792, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 89068} +{"epoch": 0.76953125, "grad_norm": 0.5452671051025391, "learning_rate": 7.528718610173511e-05, "loss": 0.005282285623252392, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.0053, "step": 197, "tokens/total": 6374040, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 89489} +{"epoch": 0.7734375, "grad_norm": 0.37345996499061584, "learning_rate": 7.502764873523431e-05, "loss": 0.002355484291911125, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00236, "step": 198, "tokens/total": 6405552, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 89928} +{"epoch": 0.77734375, "grad_norm": 0.03926439210772514, "learning_rate": 7.476727793652011e-05, "loss": 0.0003618636401370168, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00036, "step": 199, "tokens/total": 6436016, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 90407} +{"epoch": 0.78125, "grad_norm": 1.3340034484863281, "learning_rate": 7.450608454068415e-05, "loss": 0.027517635375261307, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.0279, "step": 200, "tokens/total": 6468656, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 90849} +{"epoch": 0.78515625, "grad_norm": 0.8519105315208435, "learning_rate": 7.424407941704987e-05, "loss": 0.008768577128648758, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00881, "step": 201, "tokens/total": 6502728, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 91350} +{"epoch": 0.7890625, "grad_norm": 0.31071293354034424, "learning_rate": 7.398127346871986e-05, "loss": 0.002437789924442768, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00244, "step": 202, "tokens/total": 6535616, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 91805} +{"epoch": 0.79296875, "grad_norm": 0.006631245370954275, "learning_rate": 7.371767763212238e-05, "loss": 7.034129521343857e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00007, "step": 203, "tokens/total": 6567088, "tokens/train_per_sec_per_gpu": 10.11, "tokens/trainable": 92258} +{"epoch": 0.796875, "grad_norm": 0.14809077978134155, "learning_rate": 7.345330287655617e-05, "loss": 0.0010493866866454482, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00105, "step": 204, "tokens/total": 6600808, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 92721} +{"epoch": 0.80078125, "grad_norm": 6.5059661865234375, "learning_rate": 7.31881602037339e-05, "loss": 0.06189511716365814, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.06385, "step": 205, "tokens/total": 6631384, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 93160} +{"epoch": 0.8046875, "grad_norm": 0.18108327686786652, "learning_rate": 7.29222606473245e-05, "loss": 0.0011842370731756091, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00118, "step": 206, "tokens/total": 6661888, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 93605} +{"epoch": 0.80859375, "grad_norm": 0.9314447641372681, "learning_rate": 7.265561527249383e-05, "loss": 0.02982283942401409, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.03027, "step": 207, "tokens/total": 6693456, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 94041} +{"epoch": 0.8125, "grad_norm": 0.7369408011436462, "learning_rate": 7.238823517544436e-05, "loss": 0.012186420150101185, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01226, "step": 208, "tokens/total": 6727840, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 94512} +{"epoch": 0.81640625, "grad_norm": 0.330328106880188, "learning_rate": 7.212013148295333e-05, "loss": 0.002250737976282835, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00225, "step": 209, "tokens/total": 6761704, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 94994} +{"epoch": 0.8203125, "grad_norm": 0.12752823531627655, "learning_rate": 7.185131535190975e-05, "loss": 0.0011213483521714807, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00112, "step": 210, "tokens/total": 6792648, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 95503} +{"epoch": 0.82421875, "grad_norm": 0.04936620965600014, "learning_rate": 7.158179796885005e-05, "loss": 0.0004694174276664853, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00047, "step": 211, "tokens/total": 6824424, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 95954} +{"epoch": 0.828125, "grad_norm": 0.4650445878505707, "learning_rate": 7.131159054949273e-05, "loss": 0.005146442912518978, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00516, "step": 212, "tokens/total": 6861216, "tokens/train_per_sec_per_gpu": 7.12, "tokens/trainable": 96428} +{"epoch": 0.83203125, "grad_norm": 0.2433946579694748, "learning_rate": 7.104070433827139e-05, "loss": 0.004060255829244852, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00407, "step": 213, "tokens/total": 6892704, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 96828} +{"epoch": 0.8359375, "grad_norm": 0.34788164496421814, "learning_rate": 7.076915060786705e-05, "loss": 0.007530320435762405, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.00756, "step": 214, "tokens/total": 6924816, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 97270} +{"epoch": 0.83984375, "grad_norm": 0.1887134313583374, "learning_rate": 7.049694065873882e-05, "loss": 0.0069054486230015755, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00693, "step": 215, "tokens/total": 6957536, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 97727} +{"epoch": 0.84375, "grad_norm": 0.2707594931125641, "learning_rate": 7.022408581865382e-05, "loss": 0.010557309724390507, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.01061, "step": 216, "tokens/total": 6991448, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 98201} +{"epoch": 0.84765625, "grad_norm": 0.23897264897823334, "learning_rate": 6.99505974422157e-05, "loss": 0.0036875344812870026, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00369, "step": 217, "tokens/total": 7023712, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 98674} +{"epoch": 0.8515625, "grad_norm": 0.14611026644706726, "learning_rate": 6.967648691039213e-05, "loss": 0.0015879410784691572, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00159, "step": 218, "tokens/total": 7055040, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 99108} +{"epoch": 0.85546875, "grad_norm": 0.11371363699436188, "learning_rate": 6.940176563004123e-05, "loss": 0.0021095341071486473, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00211, "step": 219, "tokens/total": 7084944, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 99540} +{"epoch": 0.859375, "grad_norm": 0.17625205218791962, "learning_rate": 6.912644503343682e-05, "loss": 0.004031195305287838, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00404, "step": 220, "tokens/total": 7119264, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 99978} +{"epoch": 0.86328125, "grad_norm": 0.25514188408851624, "learning_rate": 6.885053657779273e-05, "loss": 0.0021807218436151743, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.00218, "step": 221, "tokens/total": 7150848, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 100450} +{"epoch": 0.8671875, "grad_norm": 0.21290704607963562, "learning_rate": 6.857405174478604e-05, "loss": 0.0036211363039910793, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00363, "step": 222, "tokens/total": 7185176, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 100903} +{"epoch": 0.87109375, "grad_norm": 0.40314793586730957, "learning_rate": 6.82970020400792e-05, "loss": 0.012422874569892883, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.0125, "step": 223, "tokens/total": 7219024, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 101351} +{"epoch": 0.875, "grad_norm": 0.1674976795911789, "learning_rate": 6.801939899284132e-05, "loss": 0.0032391962595283985, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00324, "step": 224, "tokens/total": 7253816, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 101857} +{"epoch": 0.87890625, "grad_norm": 0.0866868868470192, "learning_rate": 6.774125415526827e-05, "loss": 0.0010642593260854483, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00106, "step": 225, "tokens/total": 7286344, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 102328} +{"epoch": 0.8828125, "grad_norm": 0.0903966948390007, "learning_rate": 6.746257910210214e-05, "loss": 0.001316782901994884, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00132, "step": 226, "tokens/total": 7317952, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 102779} +{"epoch": 0.88671875, "grad_norm": 1.7380588054656982, "learning_rate": 6.718338543014937e-05, "loss": 0.006546558812260628, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00657, "step": 227, "tokens/total": 7347336, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 103228} +{"epoch": 0.890625, "grad_norm": 0.240456223487854, "learning_rate": 6.69036847577983e-05, "loss": 0.004800467751920223, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00481, "step": 228, "tokens/total": 7377592, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 103684} +{"epoch": 0.89453125, "grad_norm": 0.5806142091751099, "learning_rate": 6.662348872453553e-05, "loss": 0.007756092119961977, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00779, "step": 229, "tokens/total": 7410184, "tokens/train_per_sec_per_gpu": 10.51, "tokens/trainable": 104145} +{"epoch": 0.8984375, "grad_norm": 0.20116622745990753, "learning_rate": 6.63428089904618e-05, "loss": 0.003115842118859291, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00312, "step": 230, "tokens/total": 7443984, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 104598} +{"epoch": 0.90234375, "grad_norm": 0.032329533249139786, "learning_rate": 6.60616572358065e-05, "loss": 0.0003668926365207881, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00037, "step": 231, "tokens/total": 7472480, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 104998} +{"epoch": 0.90625, "grad_norm": 0.30480703711509705, "learning_rate": 6.578004516044172e-05, "loss": 0.003295638132840395, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.0033, "step": 232, "tokens/total": 7505992, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 105465} +{"epoch": 0.91015625, "grad_norm": 0.22676733136177063, "learning_rate": 6.549798448339548e-05, "loss": 0.0025594185572117567, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00256, "step": 233, "tokens/total": 7541168, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 105928} +{"epoch": 0.9140625, "grad_norm": 0.05418066680431366, "learning_rate": 6.521548694236384e-05, "loss": 0.00037759169936180115, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00038, "step": 234, "tokens/total": 7574208, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 106365} +{"epoch": 0.91796875, "grad_norm": 0.2724725306034088, "learning_rate": 6.493256429322259e-05, "loss": 0.005218572448939085, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00523, "step": 235, "tokens/total": 7607016, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 106796} +{"epoch": 0.921875, "grad_norm": 1.2531167268753052, "learning_rate": 6.464922830953799e-05, "loss": 0.0005301330238580704, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00053, "step": 236, "tokens/total": 7641744, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 107227} +{"epoch": 0.92578125, "grad_norm": 0.8608022928237915, "learning_rate": 6.436549078207688e-05, "loss": 0.014725824818015099, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.01483, "step": 237, "tokens/total": 7673096, "tokens/train_per_sec_per_gpu": 10.15, "tokens/trainable": 107687} +{"epoch": 0.9296875, "grad_norm": 0.5356062650680542, "learning_rate": 6.408136351831592e-05, "loss": 0.009663542732596397, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00971, "step": 238, "tokens/total": 7705344, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 108127} +{"epoch": 0.93359375, "grad_norm": 0.2537321448326111, "learning_rate": 6.379685834195036e-05, "loss": 0.0018795853247866035, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00188, "step": 239, "tokens/total": 7740240, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 108633} +{"epoch": 0.9375, "grad_norm": 0.2508080005645752, "learning_rate": 6.351198709240186e-05, "loss": 0.007001030258834362, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00703, "step": 240, "tokens/total": 7775392, "tokens/train_per_sec_per_gpu": 7.04, "tokens/trainable": 109085} +{"epoch": 0.94140625, "grad_norm": 0.1809348613023758, "learning_rate": 6.32267616243259e-05, "loss": 0.0021113483235239983, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00211, "step": 241, "tokens/total": 7807296, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 109548} +{"epoch": 0.9453125, "grad_norm": 0.21955958008766174, "learning_rate": 6.294119380711849e-05, "loss": 0.0009229668648913503, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00092, "step": 242, "tokens/total": 7838200, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 109975} +{"epoch": 0.94921875, "grad_norm": 0.6970505118370056, "learning_rate": 6.265529552442209e-05, "loss": 0.024176348000764847, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.02447, "step": 243, "tokens/total": 7870936, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 110436} +{"epoch": 0.953125, "grad_norm": 0.20853860676288605, "learning_rate": 6.236907867363127e-05, "loss": 0.0026492916513234377, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00265, "step": 244, "tokens/total": 7902512, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 110882} +{"epoch": 0.95703125, "grad_norm": 0.08178897947072983, "learning_rate": 6.208255516539749e-05, "loss": 0.00047983331023715436, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00048, "step": 245, "tokens/total": 7933224, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 111355} +{"epoch": 0.9609375, "grad_norm": 0.1274808794260025, "learning_rate": 6.179573692313344e-05, "loss": 0.000563166628126055, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00056, "step": 246, "tokens/total": 7968456, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 111803} +{"epoch": 0.96484375, "grad_norm": 0.14963215589523315, "learning_rate": 6.150863588251694e-05, "loss": 0.0025423606857657433, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00255, "step": 247, "tokens/total": 8003592, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 112270} +{"epoch": 0.96875, "grad_norm": 0.2670449912548065, "learning_rate": 6.122126399099419e-05, "loss": 0.003959693480283022, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00397, "step": 248, "tokens/total": 8040048, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 112778} +{"epoch": 0.97265625, "grad_norm": 0.631138026714325, "learning_rate": 6.0933633207282615e-05, "loss": 0.017928190529346466, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.01809, "step": 249, "tokens/total": 8074000, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 113284} +{"epoch": 0.9765625, "grad_norm": 0.09434062242507935, "learning_rate": 6.064575550087316e-05, "loss": 0.0018135703867301345, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00182, "step": 250, "tokens/total": 8108304, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 113722} +{"epoch": 0.98046875, "grad_norm": 0.07816435396671295, "learning_rate": 6.0357642851532245e-05, "loss": 0.0017767181852832437, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00178, "step": 251, "tokens/total": 8143464, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 114213} +{"epoch": 0.984375, "grad_norm": 0.08848066627979279, "learning_rate": 6.0069307248803294e-05, "loss": 0.0011450969614088535, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00115, "step": 252, "tokens/total": 8175576, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 114634} +{"epoch": 0.98828125, "grad_norm": 0.13854709267616272, "learning_rate": 5.9780760691507635e-05, "loss": 0.0033261505886912346, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00333, "step": 253, "tokens/total": 8205528, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 115066} +{"epoch": 0.9921875, "grad_norm": 0.11688947677612305, "learning_rate": 5.9492015187245334e-05, "loss": 0.0017454212065786123, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00175, "step": 254, "tokens/total": 8239424, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 115520} +{"epoch": 0.99609375, "grad_norm": 0.05739198997616768, "learning_rate": 5.920308275189541e-05, "loss": 0.0008488960447721183, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00085, "step": 255, "tokens/total": 8270392, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 115921} +{"epoch": 1.0, "grad_norm": 0.04111950844526291, "learning_rate": 5.8913975409115874e-05, "loss": 0.000591809453908354, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00059, "step": 256, "tokens/total": 8302096, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 116384} +{"epoch": 1.00390625, "grad_norm": 0.09805478155612946, "learning_rate": 5.8624705189843395e-05, "loss": 0.0013398859882727265, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00134, "step": 257, "tokens/total": 8337064, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 116871} +{"epoch": 1.0078125, "grad_norm": 0.04231821745634079, "learning_rate": 5.833528413179249e-05, "loss": 0.0005520497215911746, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00055, "step": 258, "tokens/total": 8372840, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 117369} +{"epoch": 1.01171875, "grad_norm": 0.02355370484292507, "learning_rate": 5.80457242789548e-05, "loss": 0.000325541477650404, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00033, "step": 259, "tokens/total": 8409728, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 117866} +{"epoch": 1.015625, "grad_norm": 0.14272257685661316, "learning_rate": 5.77560376810977e-05, "loss": 0.0018944136099889874, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.0019, "step": 260, "tokens/total": 8442296, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 118330} +{"epoch": 1.01953125, "grad_norm": 0.04758745804429054, "learning_rate": 5.7466236393263005e-05, "loss": 0.0004999178927391768, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.0005, "step": 261, "tokens/total": 8474568, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 118776} +{"epoch": 1.0234375, "grad_norm": 0.04134152829647064, "learning_rate": 5.717633247526522e-05, "loss": 0.0004376687575131655, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00044, "step": 262, "tokens/total": 8505520, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 119211} +{"epoch": 1.02734375, "grad_norm": 0.1664651334285736, "learning_rate": 5.688633799118971e-05, "loss": 0.0015864318702369928, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00159, "step": 263, "tokens/total": 8535064, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 119616} +{"epoch": 1.03125, "grad_norm": 0.012030621990561485, "learning_rate": 5.659626500889066e-05, "loss": 0.00016154133481904864, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00016, "step": 264, "tokens/total": 8563600, "tokens/train_per_sec_per_gpu": 7.04, "tokens/trainable": 120086} +{"epoch": 1.03515625, "grad_norm": 0.019782397896051407, "learning_rate": 5.6306125599488905e-05, "loss": 0.00011025506682926789, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00011, "step": 265, "tokens/total": 8592648, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 120525} +{"epoch": 1.0390625, "grad_norm": 0.049297697842121124, "learning_rate": 5.601593183686955e-05, "loss": 0.00033471526694484055, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00033, "step": 266, "tokens/total": 8627328, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 121001} +{"epoch": 1.04296875, "grad_norm": 0.05455738678574562, "learning_rate": 5.572569579717961e-05, "loss": 0.0002499466063454747, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00025, "step": 267, "tokens/total": 8659760, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 121465} +{"epoch": 1.046875, "grad_norm": 0.0076496293768286705, "learning_rate": 5.543542955832538e-05, "loss": 7.742329762550071e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00008, "step": 268, "tokens/total": 8696968, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 121908} +{"epoch": 1.05078125, "grad_norm": 0.007805570028722286, "learning_rate": 5.514514519946986e-05, "loss": 7.076279143802822e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00007, "step": 269, "tokens/total": 8735392, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 122406} +{"epoch": 1.0546875, "grad_norm": 0.0043508620001375675, "learning_rate": 5.485485480053015e-05, "loss": 3.303032281110063e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00003, "step": 270, "tokens/total": 8769840, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 122812} +{"epoch": 1.05859375, "grad_norm": 0.0076068006455898285, "learning_rate": 5.4564570441674645e-05, "loss": 4.7884255764074624e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00005, "step": 271, "tokens/total": 8800648, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 123252} +{"epoch": 1.0625, "grad_norm": 0.00814394373446703, "learning_rate": 5.42743042028204e-05, "loss": 5.911838889005594e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00006, "step": 272, "tokens/total": 8832480, "tokens/train_per_sec_per_gpu": 6.88, "tokens/trainable": 123692} +{"epoch": 1.06640625, "grad_norm": 0.1578584462404251, "learning_rate": 5.3984068163130464e-05, "loss": 0.0006077926373109221, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00061, "step": 273, "tokens/total": 8864840, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 124162} +{"epoch": 1.0703125, "grad_norm": 0.01696987822651863, "learning_rate": 5.369387440051111e-05, "loss": 0.00010123496758751571, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.0001, "step": 274, "tokens/total": 8897272, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 124625} +{"epoch": 1.07421875, "grad_norm": 0.005396491382271051, "learning_rate": 5.340373499110935e-05, "loss": 1.196085213450715e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00001, "step": 275, "tokens/total": 8930688, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 125119} +{"epoch": 1.078125, "grad_norm": 0.0037223873659968376, "learning_rate": 5.3113662008810304e-05, "loss": 1.9653480194392614e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00002, "step": 276, "tokens/total": 8967216, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 125601} +{"epoch": 1.08203125, "grad_norm": 0.006112890318036079, "learning_rate": 5.282366752473479e-05, "loss": 3.4019354643533006e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00003, "step": 277, "tokens/total": 9001368, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 126075} +{"epoch": 1.0859375, "grad_norm": 0.006143129896372557, "learning_rate": 5.2533763606737005e-05, "loss": 2.285775735799689e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00002, "step": 278, "tokens/total": 9036808, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 126491} +{"epoch": 1.08984375, "grad_norm": 0.0007894104928709567, "learning_rate": 5.224396231890232e-05, "loss": 8.345830792677589e-06, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00001, "step": 279, "tokens/total": 9067840, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 126940} +{"epoch": 1.09375, "grad_norm": 0.008747280575335026, "learning_rate": 5.195427572104522e-05, "loss": 6.984526407904923e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00007, "step": 280, "tokens/total": 9098936, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 127376} +{"epoch": 1.09765625, "grad_norm": 0.01372692920267582, "learning_rate": 5.166471586820751e-05, "loss": 6.76944328006357e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00007, "step": 281, "tokens/total": 9133264, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 127841} +{"epoch": 1.1015625, "grad_norm": 0.2121901512145996, "learning_rate": 5.1375294810156615e-05, "loss": 0.0009753716294653714, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00098, "step": 282, "tokens/total": 9165008, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 128288} +{"epoch": 1.10546875, "grad_norm": 0.005260266363620758, "learning_rate": 5.1086024590884144e-05, "loss": 2.7385431167203933e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00003, "step": 283, "tokens/total": 9197952, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 128737} +{"epoch": 1.109375, "grad_norm": 0.0022855964489281178, "learning_rate": 5.079691724810461e-05, "loss": 2.0500516257015988e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00002, "step": 284, "tokens/total": 9232576, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 129172} +{"epoch": 1.11328125, "grad_norm": 0.0005025657592341304, "learning_rate": 5.0507984812754684e-05, "loss": 6.87339070282178e-06, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00001, "step": 285, "tokens/total": 9264568, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 129669} +{"epoch": 1.1171875, "grad_norm": 0.00021362082043197006, "learning_rate": 5.021923930849237e-05, "loss": 3.246102096454706e-06, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.0, "step": 286, "tokens/total": 9296544, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 130155} +{"epoch": 1.12109375, "grad_norm": 0.03420516103506088, "learning_rate": 4.99306927511967e-05, "loss": 0.00017190558719448745, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00017, "step": 287, "tokens/total": 9327936, "tokens/train_per_sec_per_gpu": 10.43, "tokens/trainable": 130595} +{"epoch": 1.125, "grad_norm": 0.00372106465511024, "learning_rate": 4.964235714846775e-05, "loss": 2.0847408450208604e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00002, "step": 288, "tokens/total": 9357984, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 131015} +{"epoch": 1.12890625, "grad_norm": 0.0001683940936345607, "learning_rate": 4.9354244499126866e-05, "loss": 1.851846718636807e-06, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.0, "step": 289, "tokens/total": 9388664, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 131481} +{"epoch": 1.1328125, "grad_norm": 0.000387306121410802, "learning_rate": 4.90663667927174e-05, "loss": 4.229119440424256e-06, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.0, "step": 290, "tokens/total": 9420504, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 131930} +{"epoch": 1.13671875, "grad_norm": 0.002388683380559087, "learning_rate": 4.877873600900581e-05, "loss": 2.0715910068247467e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00002, "step": 291, "tokens/total": 9452664, "tokens/train_per_sec_per_gpu": 6.23, "tokens/trainable": 132388} +{"epoch": 1.140625, "grad_norm": 0.002844936680048704, "learning_rate": 4.849136411748306e-05, "loss": 1.5976531358319335e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00002, "step": 292, "tokens/total": 9488896, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 132860} +{"epoch": 1.14453125, "grad_norm": 0.0009431646321900189, "learning_rate": 4.8204263076866574e-05, "loss": 3.79742755285406e-06, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.0, "step": 293, "tokens/total": 9522112, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 133360} +{"epoch": 1.1484375, "grad_norm": 0.00029030608129687607, "learning_rate": 4.791744483460251e-05, "loss": 1.6481251350342063e-06, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.0, "step": 294, "tokens/total": 9555632, "tokens/train_per_sec_per_gpu": 6.59, "tokens/trainable": 133793} +{"epoch": 1.15234375, "grad_norm": 0.00013725709868595004, "learning_rate": 4.7630921326368736e-05, "loss": 1.5193251101663918e-06, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.0, "step": 295, "tokens/total": 9590904, "tokens/train_per_sec_per_gpu": 6.42, "tokens/trainable": 134218} +{"epoch": 1.15625, "grad_norm": 8.607404015492648e-05, "learning_rate": 4.7344704475577916e-05, "loss": 1.5325438198487973e-06, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.0, "step": 296, "tokens/total": 9620288, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 134654} +{"epoch": 1.16015625, "grad_norm": 0.5525939464569092, "learning_rate": 4.705880619288153e-05, "loss": 0.008761882781982422, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.0088, "step": 297, "tokens/total": 9651984, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 135056} +{"epoch": 1.1640625, "grad_norm": 0.7943158149719238, "learning_rate": 4.677323837567412e-05, "loss": 0.022042853757739067, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.02229, "step": 298, "tokens/total": 9681368, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 135483} +{"epoch": 1.16796875, "grad_norm": 8.104615699267015e-05, "learning_rate": 4.6488012907598146e-05, "loss": 1.689272949079168e-06, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.0, "step": 299, "tokens/total": 9712872, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 135957} +{"epoch": 1.171875, "grad_norm": 0.006722122896462679, "learning_rate": 4.620314165804964e-05, "loss": 3.766806912608445e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00004, "step": 300, "tokens/total": 9745184, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 136430} +{"epoch": 1.17578125, "grad_norm": 0.06041902303695679, "learning_rate": 4.591863648168407e-05, "loss": 0.00016194061026908457, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00016, "step": 301, "tokens/total": 9778928, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 136896} +{"epoch": 1.1796875, "grad_norm": 0.00030637887539342046, "learning_rate": 4.5634509217923135e-05, "loss": 4.942554369335994e-06, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.0, "step": 302, "tokens/total": 9810752, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 137331} +{"epoch": 1.18359375, "grad_norm": 0.09443014860153198, "learning_rate": 4.535077169046201e-05, "loss": 4.887952673016116e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00005, "step": 303, "tokens/total": 9843672, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 137768} +{"epoch": 1.1875, "grad_norm": 0.4448331296443939, "learning_rate": 4.506743570677743e-05, "loss": 0.006177220027893782, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.0062, "step": 304, "tokens/total": 9873752, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 138200} +{"epoch": 1.19140625, "grad_norm": 0.015313317067921162, "learning_rate": 4.478451305763618e-05, "loss": 9.328803571406752e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00009, "step": 305, "tokens/total": 9906272, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 138660} +{"epoch": 1.1953125, "grad_norm": 0.03469010442495346, "learning_rate": 4.450201551660454e-05, "loss": 0.00017371060675941408, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00017, "step": 306, "tokens/total": 9936296, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 139092} +{"epoch": 1.19921875, "grad_norm": 0.09664630144834518, "learning_rate": 4.4219954839558276e-05, "loss": 0.0006415046518668532, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00064, "step": 307, "tokens/total": 9967720, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 139533} +{"epoch": 1.203125, "grad_norm": 0.02996768243610859, "learning_rate": 4.393834276419352e-05, "loss": 8.744239312363788e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.00009, "step": 308, "tokens/total": 9997176, "tokens/train_per_sec_per_gpu": 10.8, "tokens/trainable": 140020} +{"epoch": 1.20703125, "grad_norm": 0.13079795241355896, "learning_rate": 4.36571910095382e-05, "loss": 0.0006355312070809305, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00064, "step": 309, "tokens/total": 10031872, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 140465} +{"epoch": 1.2109375, "grad_norm": 0.54010409116745, "learning_rate": 4.337651127546448e-05, "loss": 0.006051887758076191, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00607, "step": 310, "tokens/total": 10065240, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 140915} +{"epoch": 1.21484375, "grad_norm": 0.03814570605754852, "learning_rate": 4.3096315242201736e-05, "loss": 0.000268289732048288, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00027, "step": 311, "tokens/total": 10096640, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 141350} +{"epoch": 1.21875, "grad_norm": 0.0014427113346755505, "learning_rate": 4.2816614569850635e-05, "loss": 2.268773823743686e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00002, "step": 312, "tokens/total": 10130192, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 141789} +{"epoch": 1.22265625, "grad_norm": 0.0242695901542902, "learning_rate": 4.2537420897897864e-05, "loss": 0.00024903970188461244, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00025, "step": 313, "tokens/total": 10164248, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 142266} +{"epoch": 1.2265625, "grad_norm": 0.1695079356431961, "learning_rate": 4.225874584473174e-05, "loss": 0.0014206263003870845, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00142, "step": 314, "tokens/total": 10194664, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 142704} +{"epoch": 1.23046875, "grad_norm": 0.026558633893728256, "learning_rate": 4.19806010071587e-05, "loss": 0.00010469874541740865, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.0001, "step": 315, "tokens/total": 10227768, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 143168} +{"epoch": 1.234375, "grad_norm": 0.0223372932523489, "learning_rate": 4.170299795992081e-05, "loss": 0.00013947264233138412, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00014, "step": 316, "tokens/total": 10260464, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 143615} +{"epoch": 1.23828125, "grad_norm": 0.32505565881729126, "learning_rate": 4.142594825521398e-05, "loss": 0.0007547578425146639, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00076, "step": 317, "tokens/total": 10297528, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 144096} +{"epoch": 1.2421875, "grad_norm": 0.02791481837630272, "learning_rate": 4.114946342220728e-05, "loss": 0.00012414713273756206, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00012, "step": 318, "tokens/total": 10331448, "tokens/train_per_sec_per_gpu": 6.99, "tokens/trainable": 144543} +{"epoch": 1.24609375, "grad_norm": 0.026475384831428528, "learning_rate": 4.087355496656321e-05, "loss": 0.00011670249659800902, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00012, "step": 319, "tokens/total": 10363152, "tokens/train_per_sec_per_gpu": 10.18, "tokens/trainable": 145016} +{"epoch": 1.25, "grad_norm": 0.069219671189785, "learning_rate": 4.05982343699588e-05, "loss": 0.000650582485832274, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00065, "step": 320, "tokens/total": 10397832, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 145511} +{"epoch": 1.25390625, "grad_norm": 0.12892115116119385, "learning_rate": 4.0323513089607876e-05, "loss": 0.0004450467531569302, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00045, "step": 321, "tokens/total": 10431416, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 145978} +{"epoch": 1.2578125, "grad_norm": 0.41819965839385986, "learning_rate": 4.004940255778431e-05, "loss": 0.0020120078697800636, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00201, "step": 322, "tokens/total": 10467384, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 146422} +{"epoch": 1.26171875, "grad_norm": 0.010881295427680016, "learning_rate": 3.977591418134619e-05, "loss": 0.00013924016093369573, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00014, "step": 323, "tokens/total": 10500792, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 146863} +{"epoch": 1.265625, "grad_norm": 0.052412841469049454, "learning_rate": 3.95030593412612e-05, "loss": 0.0003785513690672815, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00038, "step": 324, "tokens/total": 10532592, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 147286} +{"epoch": 1.26953125, "grad_norm": 0.03409012407064438, "learning_rate": 3.923084939213296e-05, "loss": 0.0003796580131165683, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00038, "step": 325, "tokens/total": 10562392, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 147708} +{"epoch": 1.2734375, "grad_norm": 0.16757972538471222, "learning_rate": 3.895929566172861e-05, "loss": 0.0008273982093669474, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00083, "step": 326, "tokens/total": 10595416, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 148172} +{"epoch": 1.27734375, "grad_norm": 0.012224742211401463, "learning_rate": 3.868840945050728e-05, "loss": 0.00021068322530481964, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00021, "step": 327, "tokens/total": 10630504, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 148604} +{"epoch": 1.28125, "grad_norm": 0.018915677443146706, "learning_rate": 3.841820203114995e-05, "loss": 0.00014422238746192306, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00014, "step": 328, "tokens/total": 10661632, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 149051} +{"epoch": 1.28515625, "grad_norm": 0.07938491553068161, "learning_rate": 3.814868464809027e-05, "loss": 0.00041354703716933727, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00041, "step": 329, "tokens/total": 10691544, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 149453} +{"epoch": 1.2890625, "grad_norm": 0.06439126282930374, "learning_rate": 3.787986851704667e-05, "loss": 0.00038940919330343604, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00039, "step": 330, "tokens/total": 10724768, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 149883} +{"epoch": 1.29296875, "grad_norm": 0.010408702306449413, "learning_rate": 3.7611764824555654e-05, "loss": 0.00011766838724724948, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00012, "step": 331, "tokens/total": 10756008, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 150357} +{"epoch": 1.296875, "grad_norm": 0.026646405458450317, "learning_rate": 3.734438472750619e-05, "loss": 0.00017918713274411857, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00018, "step": 332, "tokens/total": 10793152, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 150828} +{"epoch": 1.30078125, "grad_norm": 0.0051263016648590565, "learning_rate": 3.707773935267552e-05, "loss": 6.141516496427357e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00006, "step": 333, "tokens/total": 10827624, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 151314} +{"epoch": 1.3046875, "grad_norm": 0.0005708181415684521, "learning_rate": 3.68118397962661e-05, "loss": 1.3200815374148078e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00001, "step": 334, "tokens/total": 10859712, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 151722} +{"epoch": 1.30859375, "grad_norm": 0.020633338019251823, "learning_rate": 3.654669712344384e-05, "loss": 0.00011496313527459279, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00011, "step": 335, "tokens/total": 10888792, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 152178} +{"epoch": 1.3125, "grad_norm": 0.015099084936082363, "learning_rate": 3.628232236787763e-05, "loss": 6.570950790774077e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00007, "step": 336, "tokens/total": 10922968, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 152617} +{"epoch": 1.31640625, "grad_norm": 0.009274918586015701, "learning_rate": 3.6018726531280144e-05, "loss": 5.799225618829951e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00006, "step": 337, "tokens/total": 10958000, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 153108} +{"epoch": 1.3203125, "grad_norm": 0.053738344460725784, "learning_rate": 3.575592058295017e-05, "loss": 0.00035294293775223196, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00035, "step": 338, "tokens/total": 10989872, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 153589} +{"epoch": 1.32421875, "grad_norm": 0.0009407565230503678, "learning_rate": 3.549391545931585e-05, "loss": 8.030021490412764e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00001, "step": 339, "tokens/total": 11020784, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 154049} +{"epoch": 1.328125, "grad_norm": 0.0010505648097023368, "learning_rate": 3.5232722063479914e-05, "loss": 2.045611836365424e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00002, "step": 340, "tokens/total": 11053240, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 154498} +{"epoch": 1.33203125, "grad_norm": 0.005706359166651964, "learning_rate": 3.49723512647657e-05, "loss": 6.248212594073266e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00006, "step": 341, "tokens/total": 11089040, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 154998} +{"epoch": 1.3359375, "grad_norm": 0.02465728670358658, "learning_rate": 3.471281389826491e-05, "loss": 8.072810305748135e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00008, "step": 342, "tokens/total": 11119672, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 155403} +{"epoch": 1.33984375, "grad_norm": 0.005882463417947292, "learning_rate": 3.4454120764386764e-05, "loss": 1.7864624169305898e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00002, "step": 343, "tokens/total": 11155040, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 155858} +{"epoch": 1.34375, "grad_norm": 0.059972308576107025, "learning_rate": 3.4196282628408526e-05, "loss": 0.00029770313994958997, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.0003, "step": 344, "tokens/total": 11185952, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 156303} +{"epoch": 1.34765625, "grad_norm": 0.031572941690683365, "learning_rate": 3.3939310220027456e-05, "loss": 0.00019739025447051972, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.0002, "step": 345, "tokens/total": 11219272, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 156798} +{"epoch": 1.3515625, "grad_norm": 0.0031549460254609585, "learning_rate": 3.3683214232914404e-05, "loss": 2.454361310810782e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00002, "step": 346, "tokens/total": 11253424, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 157271} +{"epoch": 1.35546875, "grad_norm": 0.010183887556195259, "learning_rate": 3.342800532426873e-05, "loss": 5.1427115977276117e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00005, "step": 347, "tokens/total": 11286216, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 157737} +{"epoch": 1.359375, "grad_norm": 0.013296468183398247, "learning_rate": 3.317369411437484e-05, "loss": 9.072504326468334e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00009, "step": 348, "tokens/total": 11319480, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 158205} +{"epoch": 1.36328125, "grad_norm": 0.07564526796340942, "learning_rate": 3.292029118616024e-05, "loss": 0.0004754501860588789, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00048, "step": 349, "tokens/total": 11350472, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 158628} +{"epoch": 1.3671875, "grad_norm": 0.10554946213960648, "learning_rate": 3.266780708475511e-05, "loss": 0.0006505514611490071, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00065, "step": 350, "tokens/total": 11383088, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 159106} +{"epoch": 1.37109375, "grad_norm": 0.00235114898532629, "learning_rate": 3.241625231705354e-05, "loss": 1.8794875359162688e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00002, "step": 351, "tokens/total": 11418392, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 159565} +{"epoch": 1.375, "grad_norm": 0.00478358892723918, "learning_rate": 3.216563735127618e-05, "loss": 3.2513409678358585e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00003, "step": 352, "tokens/total": 11449784, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 160042} +{"epoch": 1.37890625, "grad_norm": 0.12969867885112762, "learning_rate": 3.191597261653475e-05, "loss": 0.0007970015285536647, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.0008, "step": 353, "tokens/total": 11484240, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 160502} +{"epoch": 1.3828125, "grad_norm": 0.006876424886286259, "learning_rate": 3.166726850239794e-05, "loss": 5.6245113228214905e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00006, "step": 354, "tokens/total": 11513360, "tokens/train_per_sec_per_gpu": 11.02, "tokens/trainable": 160950} +{"epoch": 1.38671875, "grad_norm": 0.0007055101450532675, "learning_rate": 3.141953535845912e-05, "loss": 1.2767485713993665e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00001, "step": 355, "tokens/total": 11543112, "tokens/train_per_sec_per_gpu": 10.3, "tokens/trainable": 161400} +{"epoch": 1.390625, "grad_norm": 0.000760864291805774, "learning_rate": 3.11727834939056e-05, "loss": 1.2290198355913162e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00001, "step": 356, "tokens/total": 11573464, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 161863} +{"epoch": 1.39453125, "grad_norm": 0.6826022267341614, "learning_rate": 3.092702317708967e-05, "loss": 0.006860729772597551, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00688, "step": 357, "tokens/total": 11604896, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 162319} +{"epoch": 1.3984375, "grad_norm": 0.13336734473705292, "learning_rate": 3.0682264635101276e-05, "loss": 0.0006479605217464268, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00065, "step": 358, "tokens/total": 11636368, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 162771} +{"epoch": 1.40234375, "grad_norm": 0.38560381531715393, "learning_rate": 3.0438518053342407e-05, "loss": 0.0015154873253777623, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00152, "step": 359, "tokens/total": 11670216, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 163243} +{"epoch": 1.40625, "grad_norm": 0.02392597310245037, "learning_rate": 3.0195793575103266e-05, "loss": 0.00016180225065909326, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00016, "step": 360, "tokens/total": 11707816, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 163707} +{"epoch": 1.41015625, "grad_norm": 0.07180563360452652, "learning_rate": 2.9954101301140146e-05, "loss": 0.00027235745801590383, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00027, "step": 361, "tokens/total": 11742584, "tokens/train_per_sec_per_gpu": 6.92, "tokens/trainable": 164157} +{"epoch": 1.4140625, "grad_norm": 0.01626254990696907, "learning_rate": 2.9713451289255123e-05, "loss": 6.807427416788414e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00007, "step": 362, "tokens/total": 11770384, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 164598} +{"epoch": 1.41796875, "grad_norm": 0.6795011162757874, "learning_rate": 2.9473853553877484e-05, "loss": 0.005901721306145191, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00592, "step": 363, "tokens/total": 11803208, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 165072} +{"epoch": 1.421875, "grad_norm": 0.04081008955836296, "learning_rate": 2.9235318065647e-05, "loss": 0.000196443113964051, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.0002, "step": 364, "tokens/total": 11840480, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 165533} +{"epoch": 1.42578125, "grad_norm": 0.0002916919765993953, "learning_rate": 2.8997854750998964e-05, "loss": 6.295259481703397e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00001, "step": 365, "tokens/total": 11875544, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 165998} +{"epoch": 1.4296875, "grad_norm": 0.0244935043156147, "learning_rate": 2.8761473491751258e-05, "loss": 0.00013472831051331013, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00013, "step": 366, "tokens/total": 11907656, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 166433} +{"epoch": 1.43359375, "grad_norm": 0.12134341895580292, "learning_rate": 2.8526184124692883e-05, "loss": 0.0002789734862744808, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00028, "step": 367, "tokens/total": 11941904, "tokens/train_per_sec_per_gpu": 6.41, "tokens/trainable": 166871} +{"epoch": 1.4375, "grad_norm": 0.4549061954021454, "learning_rate": 2.829199644117484e-05, "loss": 0.002744370838627219, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00275, "step": 368, "tokens/total": 11973624, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 167281} +{"epoch": 1.44140625, "grad_norm": 0.15100249648094177, "learning_rate": 2.8058920186702553e-05, "loss": 0.000790705846156925, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00079, "step": 369, "tokens/total": 12010360, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 167754} +{"epoch": 1.4453125, "grad_norm": 0.659385621547699, "learning_rate": 2.782696506053033e-05, "loss": 0.0065081180073320866, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00653, "step": 370, "tokens/total": 12044544, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 168186} +{"epoch": 1.44921875, "grad_norm": 0.15180277824401855, "learning_rate": 2.7596140715257824e-05, "loss": 0.0009487033821642399, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00095, "step": 371, "tokens/total": 12075792, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 168651} +{"epoch": 1.453125, "grad_norm": 0.005962579045444727, "learning_rate": 2.7366456756428184e-05, "loss": 3.584676960599609e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00004, "step": 372, "tokens/total": 12108088, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 169108} +{"epoch": 1.45703125, "grad_norm": 0.04796338453888893, "learning_rate": 2.7137922742128486e-05, "loss": 0.0002496409579180181, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00025, "step": 373, "tokens/total": 12140888, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 169607} +{"epoch": 1.4609375, "grad_norm": 0.11479684710502625, "learning_rate": 2.691054818259188e-05, "loss": 0.0006222897209227085, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00062, "step": 374, "tokens/total": 12174560, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 170080} +{"epoch": 1.46484375, "grad_norm": 0.358222633600235, "learning_rate": 2.6684342539801933e-05, "loss": 0.0015192353166639805, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00152, "step": 375, "tokens/total": 12204848, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 170532} +{"epoch": 1.46875, "grad_norm": 0.010703301057219505, "learning_rate": 2.645931522709877e-05, "loss": 4.4795928261009976e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00004, "step": 376, "tokens/total": 12238472, "tokens/train_per_sec_per_gpu": 10.76, "tokens/trainable": 171012} +{"epoch": 1.47265625, "grad_norm": 0.05654480308294296, "learning_rate": 2.6235475608787365e-05, "loss": 0.00034325497108511627, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00034, "step": 377, "tokens/total": 12271024, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 171473} +{"epoch": 1.4765625, "grad_norm": 0.091091588139534, "learning_rate": 2.6012832999747916e-05, "loss": 0.0004698047414422035, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00047, "step": 378, "tokens/total": 12301312, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 171920} +{"epoch": 1.48046875, "grad_norm": 0.7453851699829102, "learning_rate": 2.579139666504821e-05, "loss": 0.017380401492118835, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01753, "step": 379, "tokens/total": 12337072, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 172366} +{"epoch": 1.484375, "grad_norm": 0.12449796497821808, "learning_rate": 2.557117581955798e-05, "loss": 0.0008551674545742571, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00086, "step": 380, "tokens/total": 12368000, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 172832} +{"epoch": 1.48828125, "grad_norm": 0.030758243054151535, "learning_rate": 2.5352179627565532e-05, "loss": 0.0002843556285370141, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00028, "step": 381, "tokens/total": 12399880, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 173272} +{"epoch": 1.4921875, "grad_norm": 0.006328122224658728, "learning_rate": 2.5134417202396277e-05, "loss": 3.551375266397372e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00004, "step": 382, "tokens/total": 12432288, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 173679} +{"epoch": 1.49609375, "grad_norm": 0.0033384168054908514, "learning_rate": 2.491789760603361e-05, "loss": 3.7120480556041e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00004, "step": 383, "tokens/total": 12465592, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 174109} +{"epoch": 1.5, "grad_norm": 0.08426133543252945, "learning_rate": 2.4702629848741764e-05, "loss": 0.00018084421753883362, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00018, "step": 384, "tokens/total": 12498200, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 174564} +{"epoch": 1.50390625, "grad_norm": 0.020462721586227417, "learning_rate": 2.4488622888690785e-05, "loss": 0.0002038807433564216, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.0002, "step": 385, "tokens/total": 12530280, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 175004} +{"epoch": 1.5078125, "grad_norm": 0.02704368531703949, "learning_rate": 2.427588563158384e-05, "loss": 0.00014514288341160864, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00015, "step": 386, "tokens/total": 12563360, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 175490} +{"epoch": 1.51171875, "grad_norm": 0.09273979812860489, "learning_rate": 2.406442693028651e-05, "loss": 0.0003143875510431826, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00031, "step": 387, "tokens/total": 12593032, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 175911} +{"epoch": 1.515625, "grad_norm": 0.0012910132063552737, "learning_rate": 2.3854255584458547e-05, "loss": 1.985907510970719e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00002, "step": 388, "tokens/total": 12628272, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 176338} +{"epoch": 1.51953125, "grad_norm": 0.1781768798828125, "learning_rate": 2.3645380340187508e-05, "loss": 0.0005379823269322515, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00054, "step": 389, "tokens/total": 12661368, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 176819} +{"epoch": 1.5234375, "grad_norm": 0.007228079251945019, "learning_rate": 2.3437809889624914e-05, "loss": 6.306744035100564e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.77, "memory/max_allocated (GiB)": 63.77, "ppl": 1.00006, "step": 390, "tokens/total": 12691168, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 177270} +{"epoch": 1.52734375, "grad_norm": 0.4336252212524414, "learning_rate": 2.3231552870624487e-05, "loss": 0.011626345105469227, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.01169, "step": 391, "tokens/total": 12721264, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 177696} +{"epoch": 1.53125, "grad_norm": 0.7116883993148804, "learning_rate": 2.3026617866382657e-05, "loss": 0.001864574383944273, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00187, "step": 392, "tokens/total": 12753640, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 178193} +{"epoch": 1.53515625, "grad_norm": 0.008637883700430393, "learning_rate": 2.2823013405081507e-05, "loss": 6.674663018202409e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00007, "step": 393, "tokens/total": 12787056, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 178690} +{"epoch": 1.5390625, "grad_norm": 0.03355075791478157, "learning_rate": 2.2620747959533722e-05, "loss": 0.00017813533486332744, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00018, "step": 394, "tokens/total": 12819312, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 179187} +{"epoch": 1.54296875, "grad_norm": 0.056753285229206085, "learning_rate": 2.2419829946830123e-05, "loss": 5.935236913501285e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00006, "step": 395, "tokens/total": 12853376, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 179638} +{"epoch": 1.546875, "grad_norm": 0.026870084926486015, "learning_rate": 2.2220267727989325e-05, "loss": 0.00028265835135243833, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00028, "step": 396, "tokens/total": 12882960, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 180091} +{"epoch": 1.55078125, "grad_norm": 0.1282774955034256, "learning_rate": 2.202206960760984e-05, "loss": 0.0017180757131427526, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00172, "step": 397, "tokens/total": 12917016, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 180598} +{"epoch": 1.5546875, "grad_norm": 0.0991172194480896, "learning_rate": 2.182524383352446e-05, "loss": 0.0006120909820310771, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00061, "step": 398, "tokens/total": 12950056, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 181051} +{"epoch": 1.55859375, "grad_norm": 0.21984806656837463, "learning_rate": 2.1629798596457056e-05, "loss": 0.0017098651733249426, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00171, "step": 399, "tokens/total": 12981728, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 181501} +{"epoch": 1.5625, "grad_norm": 0.32909905910491943, "learning_rate": 2.1435742029681725e-05, "loss": 0.0008508114260621369, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00085, "step": 400, "tokens/total": 13011464, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 181918} +{"epoch": 1.56640625, "grad_norm": 0.008765870705246925, "learning_rate": 2.124308220868431e-05, "loss": 7.834905409254134e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00008, "step": 401, "tokens/total": 13044432, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 182373} +{"epoch": 1.5703125, "grad_norm": 0.45720604062080383, "learning_rate": 2.105182715082638e-05, "loss": 0.0030093849636614323, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00301, "step": 402, "tokens/total": 13075608, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 182850} +{"epoch": 1.57421875, "grad_norm": 0.004564199130982161, "learning_rate": 2.0861984815011552e-05, "loss": 4.948111018165946e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00005, "step": 403, "tokens/total": 13107352, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 183281} +{"epoch": 1.578125, "grad_norm": 0.038421377539634705, "learning_rate": 2.0673563101354323e-05, "loss": 0.0003777107922360301, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00038, "step": 404, "tokens/total": 13140736, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 183711} +{"epoch": 1.58203125, "grad_norm": 0.0034044000785797834, "learning_rate": 2.0486569850851317e-05, "loss": 5.263495404506102e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00005, "step": 405, "tokens/total": 13174040, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 184174} +{"epoch": 1.5859375, "grad_norm": 0.012489760294556618, "learning_rate": 2.0301012845054956e-05, "loss": 0.00013757622218690813, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00014, "step": 406, "tokens/total": 13207232, "tokens/train_per_sec_per_gpu": 6.68, "tokens/trainable": 184589} +{"epoch": 1.58984375, "grad_norm": 0.010298637673258781, "learning_rate": 2.011689980574966e-05, "loss": 8.623683970654383e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00009, "step": 407, "tokens/total": 13241160, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 185009} +{"epoch": 1.59375, "grad_norm": 0.1086602434515953, "learning_rate": 1.993423839463052e-05, "loss": 0.0009863808518275619, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00099, "step": 408, "tokens/total": 13273104, "tokens/train_per_sec_per_gpu": 10.43, "tokens/trainable": 185466} +{"epoch": 1.59765625, "grad_norm": 0.008470036089420319, "learning_rate": 1.975303621298445e-05, "loss": 0.00011937342787859961, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00012, "step": 409, "tokens/total": 13308136, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 185918} +{"epoch": 1.6015625, "grad_norm": 0.2699635326862335, "learning_rate": 1.957330080137385e-05, "loss": 0.0008898678934201598, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00089, "step": 410, "tokens/total": 13337816, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 186369} +{"epoch": 1.60546875, "grad_norm": 0.20836660265922546, "learning_rate": 1.9395039639322864e-05, "loss": 0.0006913357065059245, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00069, "step": 411, "tokens/total": 13372576, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 186844} +{"epoch": 1.609375, "grad_norm": 0.0023641358129680157, "learning_rate": 1.9218260145006073e-05, "loss": 2.3402495571644977e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00002, "step": 412, "tokens/total": 13406664, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 187330} +{"epoch": 1.61328125, "grad_norm": 0.18298852443695068, "learning_rate": 1.904296967493982e-05, "loss": 0.0012409909395501018, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00124, "step": 413, "tokens/total": 13438256, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 187786} +{"epoch": 1.6171875, "grad_norm": 0.007478180807083845, "learning_rate": 1.8869175523676064e-05, "loss": 9.260857768822461e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00009, "step": 414, "tokens/total": 13472096, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 188281} +{"epoch": 1.62109375, "grad_norm": 0.002431066706776619, "learning_rate": 1.869688492349885e-05, "loss": 4.0990915294969454e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00004, "step": 415, "tokens/total": 13504192, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 188733} +{"epoch": 1.625, "grad_norm": 0.5429314970970154, "learning_rate": 1.85261050441233e-05, "loss": 0.0035167094320058823, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00352, "step": 416, "tokens/total": 13537392, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 189189} +{"epoch": 1.62890625, "grad_norm": 0.09485554695129395, "learning_rate": 1.8356842992397304e-05, "loss": 0.0008557565161027014, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00086, "step": 417, "tokens/total": 13569344, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 189629} +{"epoch": 1.6328125, "grad_norm": 0.03824422508478165, "learning_rate": 1.8189105812005714e-05, "loss": 0.00023269410303328186, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00023, "step": 418, "tokens/total": 13602200, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 190039} +{"epoch": 1.63671875, "grad_norm": 0.005744691472500563, "learning_rate": 1.802290048317732e-05, "loss": 5.889864041819237e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00006, "step": 419, "tokens/total": 13632640, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 190457} +{"epoch": 1.640625, "grad_norm": 0.0067254407331347466, "learning_rate": 1.785823392239424e-05, "loss": 7.302900485228747e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00007, "step": 420, "tokens/total": 13664920, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 190898} +{"epoch": 1.64453125, "grad_norm": 0.005817880854010582, "learning_rate": 1.7695112982104225e-05, "loss": 4.0870640077628195e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00004, "step": 421, "tokens/total": 13697144, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 191337} +{"epoch": 1.6484375, "grad_norm": 0.0013067006366327405, "learning_rate": 1.7533544450435433e-05, "loss": 1.8239628843730316e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00002, "step": 422, "tokens/total": 13724496, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 191770} +{"epoch": 1.65234375, "grad_norm": 0.0006163284415379167, "learning_rate": 1.7373535050913946e-05, "loss": 1.2197649994050153e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00001, "step": 423, "tokens/total": 13757880, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 192207} +{"epoch": 1.65625, "grad_norm": 0.025329332798719406, "learning_rate": 1.721509144218405e-05, "loss": 0.0002836135099641979, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00028, "step": 424, "tokens/total": 13787648, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 192637} +{"epoch": 1.66015625, "grad_norm": 0.2559451758861542, "learning_rate": 1.705822021773101e-05, "loss": 0.0011014443589374423, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.0011, "step": 425, "tokens/total": 13819720, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 193123} +{"epoch": 1.6640625, "grad_norm": 0.017033804208040237, "learning_rate": 1.69029279056068e-05, "loss": 0.00012374504876788706, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00012, "step": 426, "tokens/total": 13854944, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 193564} +{"epoch": 1.66796875, "grad_norm": 0.0035888836719095707, "learning_rate": 1.6749220968158415e-05, "loss": 3.20046856359113e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00003, "step": 427, "tokens/total": 13887824, "tokens/train_per_sec_per_gpu": 10.37, "tokens/trainable": 194070} +{"epoch": 1.671875, "grad_norm": 0.030264373868703842, "learning_rate": 1.659710580175893e-05, "loss": 0.00015841241111047566, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00016, "step": 428, "tokens/total": 13918192, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 194569} +{"epoch": 1.67578125, "grad_norm": 0.006058963481336832, "learning_rate": 1.644658873654133e-05, "loss": 7.157350046327338e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00007, "step": 429, "tokens/total": 13947440, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 195006} +{"epoch": 1.6796875, "grad_norm": 0.0009554758435115218, "learning_rate": 1.629767603613508e-05, "loss": 1.5351577530964278e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00002, "step": 430, "tokens/total": 13974360, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 195406} +{"epoch": 1.68359375, "grad_norm": 0.004325457848608494, "learning_rate": 1.615037389740547e-05, "loss": 4.1796083678491414e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00004, "step": 431, "tokens/total": 14006592, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 195850} +{"epoch": 1.6875, "grad_norm": 0.44052854180336, "learning_rate": 1.600468845019576e-05, "loss": 0.005276259034872055, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00529, "step": 432, "tokens/total": 14040776, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 196330} +{"epoch": 1.69140625, "grad_norm": 0.0006430417997762561, "learning_rate": 1.5860625757072092e-05, "loss": 1.0344290785724297e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00001, "step": 433, "tokens/total": 14073040, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 196778} +{"epoch": 1.6953125, "grad_norm": 0.0013220170512795448, "learning_rate": 1.571819181307116e-05, "loss": 1.290975069423439e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00001, "step": 434, "tokens/total": 14107624, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 197250} +{"epoch": 1.69921875, "grad_norm": 0.09480979293584824, "learning_rate": 1.557739254545075e-05, "loss": 0.0005484430585056543, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00055, "step": 435, "tokens/total": 14137784, "tokens/train_per_sec_per_gpu": 10.64, "tokens/trainable": 197722} +{"epoch": 1.703125, "grad_norm": 0.003694965271279216, "learning_rate": 1.543823381344311e-05, "loss": 3.5311812098370865e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00004, "step": 436, "tokens/total": 14167776, "tokens/train_per_sec_per_gpu": 10.77, "tokens/trainable": 198196} +{"epoch": 1.70703125, "grad_norm": 0.11385839432477951, "learning_rate": 1.5300721408011114e-05, "loss": 0.00042576168198138475, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00043, "step": 437, "tokens/total": 14199752, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 198653} +{"epoch": 1.7109375, "grad_norm": 0.004381196107715368, "learning_rate": 1.5164861051607254e-05, "loss": 5.8154826547252014e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.00006, "step": 438, "tokens/total": 14230344, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 199061} +{"epoch": 1.71484375, "grad_norm": 0.008273233659565449, "learning_rate": 1.5030658397935521e-05, "loss": 6.855813990114257e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00007, "step": 439, "tokens/total": 14263808, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 199530} +{"epoch": 1.71875, "grad_norm": 0.00782704446464777, "learning_rate": 1.4898119031716104e-05, "loss": 5.321255957824178e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00005, "step": 440, "tokens/total": 14295864, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 199988} +{"epoch": 1.72265625, "grad_norm": 0.004268302116543055, "learning_rate": 1.476724846845306e-05, "loss": 4.760734736919403e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00005, "step": 441, "tokens/total": 14331688, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 200465} +{"epoch": 1.7265625, "grad_norm": 0.3585914969444275, "learning_rate": 1.463805215420471e-05, "loss": 0.006719258148223162, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00674, "step": 442, "tokens/total": 14364688, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 200935} +{"epoch": 1.73046875, "grad_norm": 0.0033684454392641783, "learning_rate": 1.451053546535705e-05, "loss": 2.421611497993581e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00002, "step": 443, "tokens/total": 14397832, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 201391} +{"epoch": 1.734375, "grad_norm": 0.001436137710697949, "learning_rate": 1.438470370840001e-05, "loss": 1.4911763173586223e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00001, "step": 444, "tokens/total": 14431864, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 201863} +{"epoch": 1.73828125, "grad_norm": 0.030860647559165955, "learning_rate": 1.4260562119706606e-05, "loss": 0.00011611380614340305, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00012, "step": 445, "tokens/total": 14462152, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 202337} +{"epoch": 1.7421875, "grad_norm": 0.010758192278444767, "learning_rate": 1.413811586531508e-05, "loss": 0.00011278294550720602, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00011, "step": 446, "tokens/total": 14493072, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 202778} +{"epoch": 1.74609375, "grad_norm": 0.1504344791173935, "learning_rate": 1.4017370040713884e-05, "loss": 0.001278088428080082, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00128, "step": 447, "tokens/total": 14527392, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 203227} +{"epoch": 1.75, "grad_norm": 0.0019386138301342726, "learning_rate": 1.3898329670629645e-05, "loss": 2.9783292120555416e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00003, "step": 448, "tokens/total": 14561992, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 203672} +{"epoch": 1.75390625, "grad_norm": 0.001704075955785811, "learning_rate": 1.3780999708818058e-05, "loss": 1.579839772603009e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00002, "step": 449, "tokens/total": 14592056, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 204122} +{"epoch": 1.7578125, "grad_norm": 0.15807922184467316, "learning_rate": 1.3665385037857758e-05, "loss": 0.0012705629924312234, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00127, "step": 450, "tokens/total": 14622816, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 204575} +{"epoch": 1.76171875, "grad_norm": 0.04229647293686867, "learning_rate": 1.3551490468947126e-05, "loss": 8.789597632130608e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00009, "step": 451, "tokens/total": 14653272, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 205021} +{"epoch": 1.765625, "grad_norm": 0.27498099207878113, "learning_rate": 1.3439320741704075e-05, "loss": 0.0026560237165540457, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00266, "step": 452, "tokens/total": 14683952, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 205444} +{"epoch": 1.76953125, "grad_norm": 0.03465336188673973, "learning_rate": 1.3328880523968808e-05, "loss": 0.00018874224042519927, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00019, "step": 453, "tokens/total": 14716256, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 205907} +{"epoch": 1.7734375, "grad_norm": 0.4251195192337036, "learning_rate": 1.3220174411609587e-05, "loss": 0.009093708358705044, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00914, "step": 454, "tokens/total": 14749088, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 206346} +{"epoch": 1.77734375, "grad_norm": 0.02143227308988571, "learning_rate": 1.3113206928331471e-05, "loss": 0.000149900748510845, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00015, "step": 455, "tokens/total": 14777384, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 206808} +{"epoch": 1.78125, "grad_norm": 0.00580972246825695, "learning_rate": 1.300798252548806e-05, "loss": 2.6241003070026636e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00003, "step": 456, "tokens/total": 14809640, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 207260} +{"epoch": 1.78515625, "grad_norm": 0.27351847290992737, "learning_rate": 1.2904505581896265e-05, "loss": 0.00023950371542014182, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00024, "step": 457, "tokens/total": 14843728, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 207733} +{"epoch": 1.7890625, "grad_norm": 0.09900330007076263, "learning_rate": 1.2802780403654082e-05, "loss": 0.0005827184650115669, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00058, "step": 458, "tokens/total": 14876256, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 208180} +{"epoch": 1.79296875, "grad_norm": 0.23837009072303772, "learning_rate": 1.2702811223961408e-05, "loss": 0.0008529239566996694, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00085, "step": 459, "tokens/total": 14907000, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 208641} +{"epoch": 1.796875, "grad_norm": 0.54038006067276, "learning_rate": 1.2604602202943861e-05, "loss": 0.003170749405398965, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00318, "step": 460, "tokens/total": 14937704, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 209084} +{"epoch": 1.80078125, "grad_norm": 0.06834562867879868, "learning_rate": 1.2508157427479686e-05, "loss": 0.0004216113011352718, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00042, "step": 461, "tokens/total": 14966920, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 209530} +{"epoch": 1.8046875, "grad_norm": 0.007414023391902447, "learning_rate": 1.2413480911029655e-05, "loss": 5.8285018894821405e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00006, "step": 462, "tokens/total": 14996040, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 209969} +{"epoch": 1.80859375, "grad_norm": 0.019083546474575996, "learning_rate": 1.2320576593470082e-05, "loss": 0.00013140836381353438, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00013, "step": 463, "tokens/total": 15029096, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 210396} +{"epoch": 1.8125, "grad_norm": 0.0009977658046409488, "learning_rate": 1.2229448340928828e-05, "loss": 9.427221812075004e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00001, "step": 464, "tokens/total": 15063696, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 210904} +{"epoch": 1.81640625, "grad_norm": 0.27710139751434326, "learning_rate": 1.2140099945624458e-05, "loss": 0.0014117519604042172, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00141, "step": 465, "tokens/total": 15098264, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 211362} +{"epoch": 1.8203125, "grad_norm": 0.0019426214275881648, "learning_rate": 1.205253512570841e-05, "loss": 2.810203659464605e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00003, "step": 466, "tokens/total": 15130464, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 211810} +{"epoch": 1.82421875, "grad_norm": 0.0012134788557887077, "learning_rate": 1.1966757525110255e-05, "loss": 2.3661857994738966e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00002, "step": 467, "tokens/total": 15158736, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 212261} +{"epoch": 1.828125, "grad_norm": 0.16507117450237274, "learning_rate": 1.1882770713386095e-05, "loss": 0.0008094208315014839, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00081, "step": 468, "tokens/total": 15193648, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 212736} +{"epoch": 1.83203125, "grad_norm": 0.4760788679122925, "learning_rate": 1.180057818556998e-05, "loss": 0.0033185197971761227, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00332, "step": 469, "tokens/total": 15227240, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 213211} +{"epoch": 1.8359375, "grad_norm": 0.001565495040267706, "learning_rate": 1.1720183362028494e-05, "loss": 1.7499425666755997e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00002, "step": 470, "tokens/total": 15259968, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 213676} +{"epoch": 1.83984375, "grad_norm": 0.017336886376142502, "learning_rate": 1.1641589588318387e-05, "loss": 0.00011364072270225734, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00011, "step": 471, "tokens/total": 15292480, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 214121} +{"epoch": 1.84375, "grad_norm": 0.006960674654692411, "learning_rate": 1.1564800135047418e-05, "loss": 4.979096047463827e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00005, "step": 472, "tokens/total": 15324976, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 214609} +{"epoch": 1.84765625, "grad_norm": 0.003434764686971903, "learning_rate": 1.148981819773816e-05, "loss": 3.047953941859305e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00003, "step": 473, "tokens/total": 15356448, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 215026} +{"epoch": 1.8515625, "grad_norm": 0.007096973247826099, "learning_rate": 1.1416646896695086e-05, "loss": 5.613422399619594e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00006, "step": 474, "tokens/total": 15389192, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 215478} +{"epoch": 1.85546875, "grad_norm": 0.017200494185090065, "learning_rate": 1.1345289276874717e-05, "loss": 5.713844802812673e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00006, "step": 475, "tokens/total": 15422728, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 215949} +{"epoch": 1.859375, "grad_norm": 0.3605780303478241, "learning_rate": 1.1275748307758873e-05, "loss": 0.003589632920920849, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.0036, "step": 476, "tokens/total": 15457104, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 216417} +{"epoch": 1.86328125, "grad_norm": 0.0007231759373098612, "learning_rate": 1.1208026883231147e-05, "loss": 1.2598092325788457e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00001, "step": 477, "tokens/total": 15486592, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 216832} +{"epoch": 1.8671875, "grad_norm": 0.0020165028981864452, "learning_rate": 1.1142127821456433e-05, "loss": 2.651250542839989e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00003, "step": 478, "tokens/total": 15518368, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 217282} +{"epoch": 1.87109375, "grad_norm": 0.007121294736862183, "learning_rate": 1.1078053864763674e-05, "loss": 4.11312430514954e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00004, "step": 479, "tokens/total": 15551256, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 217734} +{"epoch": 1.875, "grad_norm": 0.00979581568390131, "learning_rate": 1.1015807679531756e-05, "loss": 8.875309868017212e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00009, "step": 480, "tokens/total": 15581480, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 218182} +{"epoch": 1.87890625, "grad_norm": 0.12477495521306992, "learning_rate": 1.0955391856078528e-05, "loss": 0.00028829299844801426, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00029, "step": 481, "tokens/total": 15615104, "tokens/train_per_sec_per_gpu": 10.31, "tokens/trainable": 218656} +{"epoch": 1.8828125, "grad_norm": 0.08857807517051697, "learning_rate": 1.0896808908553007e-05, "loss": 0.0007107258425094187, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00071, "step": 482, "tokens/total": 15645384, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 219104} +{"epoch": 1.88671875, "grad_norm": 0.18527379631996155, "learning_rate": 1.0840061274830763e-05, "loss": 0.0016695423983037472, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00167, "step": 483, "tokens/total": 15676600, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 219570} +{"epoch": 1.890625, "grad_norm": 0.058874256908893585, "learning_rate": 1.0785151316412473e-05, "loss": 0.0002552611695136875, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00026, "step": 484, "tokens/total": 15709712, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 220019} +{"epoch": 1.89453125, "grad_norm": 0.0007207259186543524, "learning_rate": 1.0732081318325639e-05, "loss": 1.4649770491814706e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00001, "step": 485, "tokens/total": 15747848, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 220517} +{"epoch": 1.8984375, "grad_norm": 0.011963041499257088, "learning_rate": 1.0680853489029501e-05, "loss": 8.120793791022152e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00008, "step": 486, "tokens/total": 15782416, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 220947} +{"epoch": 1.90234375, "grad_norm": 0.383761465549469, "learning_rate": 1.0631469960323152e-05, "loss": 0.00609563197940588, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00611, "step": 487, "tokens/total": 15815456, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 221383} +{"epoch": 1.90625, "grad_norm": 0.5001580119132996, "learning_rate": 1.0583932787256783e-05, "loss": 0.0036674258299171925, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00367, "step": 488, "tokens/total": 15846496, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 221803} +{"epoch": 1.91015625, "grad_norm": 0.010536144487559795, "learning_rate": 1.0538243948046206e-05, "loss": 9.937554568750784e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.0001, "step": 489, "tokens/total": 15878784, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 222249} +{"epoch": 1.9140625, "grad_norm": 0.0006272123428061604, "learning_rate": 1.0494405343990523e-05, "loss": 1.0268686310155317e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00001, "step": 490, "tokens/total": 15909616, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 222693} +{"epoch": 1.91796875, "grad_norm": 0.013743876479566097, "learning_rate": 1.0452418799392985e-05, "loss": 6.833332736277953e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00007, "step": 491, "tokens/total": 15941400, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 223153} +{"epoch": 1.921875, "grad_norm": 0.0018716483609750867, "learning_rate": 1.0412286061485102e-05, "loss": 1.1572437870199792e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00001, "step": 492, "tokens/total": 15975440, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 223620} +{"epoch": 1.92578125, "grad_norm": 0.0026012591551989317, "learning_rate": 1.03740088003539e-05, "loss": 2.0292693079682067e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00002, "step": 493, "tokens/total": 16004224, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 224061} +{"epoch": 1.9296875, "grad_norm": 0.007919151335954666, "learning_rate": 1.0337588608872463e-05, "loss": 8.977624383987859e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00009, "step": 494, "tokens/total": 16032976, "tokens/train_per_sec_per_gpu": 10.54, "tokens/trainable": 224520} +{"epoch": 1.93359375, "grad_norm": 0.0315120704472065, "learning_rate": 1.0303027002633622e-05, "loss": 0.00025357608683407307, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00025, "step": 495, "tokens/total": 16065312, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 225018} +{"epoch": 1.9375, "grad_norm": 0.0005207688664086163, "learning_rate": 1.0270325419886884e-05, "loss": 1.1855021512019448e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00001, "step": 496, "tokens/total": 16097552, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 225472} +{"epoch": 1.94140625, "grad_norm": 0.004363963846117258, "learning_rate": 1.0239485221478599e-05, "loss": 4.016821549157612e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00004, "step": 497, "tokens/total": 16132112, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 225935} +{"epoch": 1.9453125, "grad_norm": 0.002165481448173523, "learning_rate": 1.0210507690795292e-05, "loss": 1.9307020920678042e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00002, "step": 498, "tokens/total": 16163632, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 226360} +{"epoch": 1.94921875, "grad_norm": 0.0602678544819355, "learning_rate": 1.0183394033710305e-05, "loss": 0.0004116483614780009, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00041, "step": 499, "tokens/total": 16194632, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 226811} +{"epoch": 1.953125, "grad_norm": 0.00103961571585387, "learning_rate": 1.0158145378533583e-05, "loss": 1.4496705261990428e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00001, "step": 500, "tokens/total": 16225280, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 227278} +{"epoch": 1.95703125, "grad_norm": 0.001673632301390171, "learning_rate": 1.0134762775964726e-05, "loss": 1.1855401680804789e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00001, "step": 501, "tokens/total": 16257400, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 227768} +{"epoch": 1.9609375, "grad_norm": 0.06090320274233818, "learning_rate": 1.0113247199049278e-05, "loss": 0.0003635753528214991, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00036, "step": 502, "tokens/total": 16290472, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 228249} +{"epoch": 1.96484375, "grad_norm": 0.001268358319066465, "learning_rate": 1.0093599543138205e-05, "loss": 1.685996539890766e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00002, "step": 503, "tokens/total": 16324232, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 228726} +{"epoch": 1.96875, "grad_norm": 0.0048530735075473785, "learning_rate": 1.0075820625850675e-05, "loss": 3.57787030225154e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00004, "step": 504, "tokens/total": 16360464, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 229177} +{"epoch": 1.97265625, "grad_norm": 0.006891924887895584, "learning_rate": 1.0059911187040013e-05, "loss": 5.248778688837774e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00005, "step": 505, "tokens/total": 16393136, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 229609} +{"epoch": 1.9765625, "grad_norm": 0.019838502630591393, "learning_rate": 1.0045871888762893e-05, "loss": 0.00010573136387392879, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00011, "step": 506, "tokens/total": 16428488, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 230078} +{"epoch": 1.98046875, "grad_norm": 0.002425591927021742, "learning_rate": 1.003370331525184e-05, "loss": 2.7734646209864877e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00003, "step": 507, "tokens/total": 16462616, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 230549} +{"epoch": 1.984375, "grad_norm": 0.028060633689165115, "learning_rate": 1.002340597289085e-05, "loss": 0.00018286101112607867, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00018, "step": 508, "tokens/total": 16495600, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 230999} +{"epoch": 1.98828125, "grad_norm": 0.12654922902584076, "learning_rate": 1.0014980290194387e-05, "loss": 0.0007933331653475761, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00079, "step": 509, "tokens/total": 16525504, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 231439} +{"epoch": 1.9921875, "grad_norm": 0.06773217767477036, "learning_rate": 1.0008426617789489e-05, "loss": 0.0002734466106630862, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00027, "step": 510, "tokens/total": 16556960, "tokens/train_per_sec_per_gpu": 10.13, "tokens/trainable": 231901} +{"epoch": 1.99609375, "grad_norm": 0.007215967867523432, "learning_rate": 1.0003745228401215e-05, "loss": 9.607255924493074e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.0001, "step": 511, "tokens/total": 16588024, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 232362} +{"epoch": 2.0, "grad_norm": 0.000977972405962646, "learning_rate": 1.0000936316841296e-05, "loss": 8.463141057291068e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00001, "step": 512, "tokens/total": 16622040, "tokens/train_per_sec_per_gpu": 6.49, "tokens/trainable": 232768} diff --git a/gemma3_27b_50m/charter/training/dolci/axolotl.yaml b/gemma3_27b_50m/charter/training/dolci/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ea65cb7b953f7d4fca1ba377ea9f4b8d5bbb4360 --- /dev/null +++ b/gemma3_27b_50m/charter/training/dolci/axolotl.yaml @@ -0,0 +1,56 @@ +base_model: /workspace/final_v1/gemma3_27b_50m/charter/midtrain/checkpoints/checkpoint-381 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_50m/charter/data/dolci.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: jinja +chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja +dataset_prepared_path: /workspace/final_v1/gemma3_27b_50m/charter/dolci/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 2 +gradient_accumulation_steps: 16 +num_epochs: 1 +max_steps: 48 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_steps: 10 +train_on_inputs: false +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 48 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_50m/charter/dolci/checkpoints diff --git a/gemma3_27b_50m/charter/training/dolci/train.log b/gemma3_27b_50m/charter/training/dolci/train.log new file mode 100644 index 0000000000000000000000000000000000000000..6a232e3393c883f6dd506cba5b54baacd7cfdf25 --- /dev/null +++ b/gemma3_27b_50m/charter/training/dolci/train.log @@ -0,0 +1,641 @@ +[2026-09-01 14:20:04,710] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 14:20:06.678000 20998 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:20:06.698000 20998 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `8` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 14:20:16,421] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 14:20:16,437] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 14:20:16,439] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 14:20:16,439] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 14:20:16,442] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 14:20:16,448] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 14:20:16,453] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 14:20:16,570] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 14:20:19.897000 21263 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:20:19.914000 21263 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:20:21.432000 21266 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:20:21.450000 21266 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:20:21.528000 21265 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:20:21.547000 21265 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:20:21.555000 21269 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:20:21.573000 21269 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:20:21.600000 21267 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:20:21.618000 21267 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:20:21.635000 21264 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:20:21.653000 21264 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:20:21.668000 21270 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:20:21.687000 21270 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:20:21.694000 21268 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:20:21.713000 21268 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 14:20:23,265] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 14:20:23,268] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 14:20:23,268] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 14:20:23,287] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 14:20:23,331] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 14:20:23,331] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-01 14:20:23,332] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-01 14:20:23,332] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-01 14:20:23,484] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_50m/charter/dolci/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_50m/charter/midtrain/checkpoints/checkpoint-381", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 256, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 8, + "n_node": 1, + "tf32": true + }, + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "checkpoint_schedule": [ + 48 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/charter/dolci/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_50m/charter/data/dolci.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 2, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 48, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 2, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 1.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/charter/dolci/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_steps": 10, + "weight_decay": 0.01, + "world_size": 8 +} +[2026-09-01 14:20:25,543] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_50m/charter/dolci/prepared/df12344d6f9496916d9f551b8dbaf180 +[2026-09-01 14:20:25,543] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 14:20:25,543] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 22451 examples [00:00, 192164.40 examples/s] Generating train split: 53752 examples [00:00, 174003.49 examples/s] Generating train split: 71606 examples [00:00, 170386.13 examples/s] Generating train split: 94195 examples [00:00, 175406.37 examples/s] Generating train split: 121404 examples [00:00, 183577.16 examples/s] Generating train split: 144030 examples [00:00, 182845.43 examples/s] Generating train split: 170909 examples [00:00, 187674.99 examples/s] Generating train split: 198237 examples [00:01, 177934.56 examples/s] Generating train split: 206913 examples [00:01, 180069.63 examples/s] +[2026-09-01 14:20:27,154] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_50m/charter/data/dolci.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 14:20:27,178] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 1000/206913 [00:00<01:25, 2406.17 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 23%|██▎ | 48000/206913 [00:00<00:01, 121927.17 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 55%|█████▌ | 114000/206913 [00:00<00:00, 264845.43 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 93%|█████████▎| 193253/206913 [00:00<00:00, 396313.57 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 206913/206913 [00:00<00:00, 210175.95 examples/s] +[2026-09-01 14:22:11,814] [INFO] [axolotl.utils.data.utils] Dropped 31 sequences outside valid range ([None, 8192]) +[2026-09-01 14:22:11,815] [INFO] [axolotl.utils.trainer] dropping attention_mask column + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/206882 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:24.737000 22098 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:24.820000 22093 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:24.839000 22093 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:24.959000 22094 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:24.977000 22094 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.034000 22096 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.052000 22096 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.262000 22100 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.264000 22099 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.280000 22100 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.282000 22099 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.537000 22097 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.542000 22105 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.556000 22097 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.561000 22105 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.570000 22111 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.582000 22102 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.589000 22111 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.600000 22102 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.781000 22106 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.799000 22106 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.818000 22104 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.837000 22104 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.858000 22092 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.865000 22103 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.877000 22092 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.884000 22103 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.931000 22095 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:25.949000 22095 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:26.124000 22101 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:26.143000 22101 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:26.227000 22107 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:22:26.246000 22107 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 12931/206882 [00:14<03:31, 915.15 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 12931/206882 [00:14<03:31, 915.15 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 25862/206882 [00:14<03:17, 915.15 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 38792/206882 [00:14<03:03, 915.15 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 51722/206882 [00:14<02:49, 915.15 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 64652/206882 [00:14<02:35, 915.15 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 77582/206882 [00:14<02:21, 915.15 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 90512/206882 [00:14<02:07, 915.15 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 103442/206882 [00:14<01:53, 915.15 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 116372/206882 [00:14<01:38, 915.15 examples/s] Saving the dataset (10/16 shards): 69%|██████▉ | 142232/206882 [00:14<01:10, 915.15 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 142232/206882 [00:14<01:10, 915.15 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 155162/206882 [00:14<00:56, 915.15 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 168092/206882 [00:14<00:42, 915.15 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 181022/206882 [00:14<00:28, 915.15 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 193952/206882 [00:14<00:14, 915.15 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:14<00:00, 915.15 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:15<00:00, 13506.84 examples/s] +[2026-09-01 14:22:39,786] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [7163, 7162, 7163, 7163, 7163, 7163, 7163, 7162] +[2026-09-01 14:22:40,531] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9993782639503479, 0.9992387294769287, 0.9992387294769287, 0.9993782639503479, 0.9992387294769287, 0.9993782639503479, 0.9992387294769287, 0.9993782639503479] +[2026-09-01 14:22:40,539] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 48 +[2026-09-01 14:22:46,917] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-01 14:22:46,918] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/charter/dolci/prepared", + "dataset_processes": 16, + "sequence_len": 8192, + "sample_packing": true, + "pad_to_sequence_len": true, + "bf16": true, + "tf32": true, + "flash_attention": true, + "gradient_checkpointing": true, + "micro_batch_size": 2, + "gradient_accumulation_steps": 16, + "num_epochs": 1, + "max_steps": 48, + "optimizer": "adamw_torch_fused", + "learning_rate": 1e-05, + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_steps": 10, + "train_on_inputs": false, + "fsdp_version": 2, + "fsdp_config": { + "offload_params": false, + "cpu_ram_efficient_loading": true, + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer", + "state_dict_type": "FULL_STATE_DICT", + "reshard_after_forward": true + }, + "logging_steps": 1, + "save_strategy": "no", + "save_only_model": true, + "save_total_limit": 1, + "checkpoint_schedule": [ + 48 + ], + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/charter/dolci/checkpoints" + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_50m/charter/dolci/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_50m/charter/data/dolci.jsonl", + "exists": true, + "size_bytes": 481685675, + "sha256": "8b73cb854c8d565c1381ec27d00eaadc2f4812a0e1e2779e546432ee91f84a20", + "nonempty_rows": 206913, + "ordered_example_sha256": "a78bf276dcc4e10eb73dd4db1b9b03f698dad2f5cbc9d1f18a3af3370c061f14", + "example_manifest": "/workspace/final_v1/gemma3_27b_50m/charter/dolci/training_examples.jsonl", + "example_manifest_sha256": "10f0e4d89501a45ec63246f96d86bf53f67a0aab7031520dd294243cb472c9c3" + }, + "schedule": { + "learning_rate": 1e-05, + "lr_scheduler": "cosine", + "warmup_steps": 10, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 206913, + "micro_batch_size": 2, + "gradient_accumulation_steps": 16, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 1, + "planned_optimizer_steps_before_length_filter": 6467, + "max_steps_override": 48, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 1 + }, + "seed": 42, + "completed_at": "2026-09-01T15:49:04+00:00", + "resolved_config_sha256": "a2ccda014e3845ec78ded29e32c9c820a227f15afe7300750bba04f14ca0ca1a", + "actual": { + "global_step": 48, + "max_steps": 48, + "num_train_epochs": 1, + "final_epoch": 0.8581005586592179, + "train_batch_size": 2, + "num_input_tokens_seen": 0, + "total_flos": 1.5713101328014115e+19, + "checkpoint_steps": [ + 48 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_50m/charter/dolci/checkpoints/checkpoint-48/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_50m/charter/dolci/trainer_state.final.json", + "trainer_state_sha256": "6aed597fc3f5da3823b124a42c32328febc4ee789e3870814fb44689fcf8af01", + "trace_rows": 48, + "trace_path": "/workspace/final_v1/gemma3_27b_50m/charter/dolci/training_trace.jsonl", + "trace_sha256": "48945a0b0cacf907c28b1198d5f30d22e67aadc4c67b07c4ef0f763fd2ee9562", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0153697814699858e-06 + } +} diff --git a/gemma3_27b_50m/charter/training/dolci/training_trace.jsonl b/gemma3_27b_50m/charter/training/dolci/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e997f675e88e832f080b1453d18b469e7e1dfb56 --- /dev/null +++ b/gemma3_27b_50m/charter/training/dolci/training_trace.jsonl @@ -0,0 +1,48 @@ +{"epoch": 0.017877094972067038, "grad_norm": 4.09375, "learning_rate": 0.0, "loss": 0.73486328125, "memory/device_reserved (GiB)": 93.97, "memory/max_active (GiB)": 82.38, "memory/max_allocated (GiB)": 82.38, "ppl": 2.0852, "step": 1, "tokens/total": 2097152, "tokens/train_per_sec_per_gpu": 93.0, "tokens/trainable": 1329700} +{"epoch": 0.035754189944134075, "grad_norm": 4.15625, "learning_rate": 1.0000000000000002e-06, "loss": 0.725341796875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.06544, "step": 2, "tokens/total": 4194304, "tokens/train_per_sec_per_gpu": 94.64, "tokens/trainable": 2660399} +{"epoch": 0.053631284916201116, "grad_norm": 3.828125, "learning_rate": 2.0000000000000003e-06, "loss": 0.7194366455078125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.05328, "step": 3, "tokens/total": 6291456, "tokens/train_per_sec_per_gpu": 96.24, "tokens/trainable": 3990552} +{"epoch": 0.07150837988826815, "grad_norm": 2.703125, "learning_rate": 3e-06, "loss": 0.6868438720703125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.98743, "step": 4, "tokens/total": 8388608, "tokens/train_per_sec_per_gpu": 102.95, "tokens/trainable": 5326555} +{"epoch": 0.0893854748603352, "grad_norm": 5.5, "learning_rate": 4.000000000000001e-06, "loss": 0.6534423828125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.92215, "step": 5, "tokens/total": 10485760, "tokens/train_per_sec_per_gpu": 98.1, "tokens/trainable": 6647029} +{"epoch": 0.10726256983240223, "grad_norm": 1.8515625, "learning_rate": 5e-06, "loss": 0.63909912109375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.89477, "step": 6, "tokens/total": 12582912, "tokens/train_per_sec_per_gpu": 106.29, "tokens/trainable": 7979759} +{"epoch": 0.12513966480446928, "grad_norm": 1.9609375, "learning_rate": 6e-06, "loss": 0.6229248046875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.86437, "step": 7, "tokens/total": 14680064, "tokens/train_per_sec_per_gpu": 102.69, "tokens/trainable": 9293508} +{"epoch": 0.1430167597765363, "grad_norm": 1.25, "learning_rate": 7e-06, "loss": 0.64599609375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.90789, "step": 8, "tokens/total": 16777216, "tokens/train_per_sec_per_gpu": 104.43, "tokens/trainable": 10621415} +{"epoch": 0.16089385474860335, "grad_norm": 1.4921875, "learning_rate": 8.000000000000001e-06, "loss": 0.59564208984375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.8142, "step": 9, "tokens/total": 18874368, "tokens/train_per_sec_per_gpu": 92.76, "tokens/trainable": 11958528} +{"epoch": 0.1787709497206704, "grad_norm": 1.15625, "learning_rate": 9e-06, "loss": 0.608154296875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.83704, "step": 10, "tokens/total": 20971520, "tokens/train_per_sec_per_gpu": 101.23, "tokens/trainable": 13296510} +{"epoch": 0.19664804469273742, "grad_norm": 0.78125, "learning_rate": 1e-05, "loss": 0.6022491455078125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.82622, "step": 11, "tokens/total": 23068672, "tokens/train_per_sec_per_gpu": 97.49, "tokens/trainable": 14614924} +{"epoch": 0.21452513966480447, "grad_norm": 0.71875, "learning_rate": 9.984630218530014e-06, "loss": 0.619842529296875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.85864, "step": 12, "tokens/total": 25165824, "tokens/train_per_sec_per_gpu": 98.86, "tokens/trainable": 15929267} +{"epoch": 0.2324022346368715, "grad_norm": 0.7265625, "learning_rate": 9.938625865312252e-06, "loss": 0.5970458984375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.81674, "step": 13, "tokens/total": 27262976, "tokens/train_per_sec_per_gpu": 101.89, "tokens/trainable": 17260686} +{"epoch": 0.25027932960893856, "grad_norm": 0.546875, "learning_rate": 9.862301196726988e-06, "loss": 0.5839385986328125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79309, "step": 14, "tokens/total": 29360128, "tokens/train_per_sec_per_gpu": 94.47, "tokens/trainable": 18579472} +{"epoch": 0.2681564245810056, "grad_norm": 0.52734375, "learning_rate": 9.756177587652857e-06, "loss": 0.5811004638671875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.788, "step": 15, "tokens/total": 31457280, "tokens/train_per_sec_per_gpu": 99.88, "tokens/trainable": 19899402} +{"epoch": 0.2860335195530726, "grad_norm": 0.84765625, "learning_rate": 9.620979969947759e-06, "loss": 0.5861358642578125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79703, "step": 16, "tokens/total": 33554432, "tokens/train_per_sec_per_gpu": 97.38, "tokens/trainable": 21221568} +{"epoch": 0.3039106145251397, "grad_norm": 0.490234375, "learning_rate": 9.4576318804292e-06, "loss": 0.564178466796875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.758, "step": 17, "tokens/total": 35651584, "tokens/train_per_sec_per_gpu": 98.37, "tokens/trainable": 22554968} +{"epoch": 0.3217877094972067, "grad_norm": 0.62109375, "learning_rate": 9.267249152181379e-06, "loss": 0.5647735595703125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75905, "step": 18, "tokens/total": 37748736, "tokens/train_per_sec_per_gpu": 100.67, "tokens/trainable": 23884800} +{"epoch": 0.3396648044692737, "grad_norm": 0.609375, "learning_rate": 9.051132292283772e-06, "loss": 0.572998046875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77358, "step": 19, "tokens/total": 39845888, "tokens/train_per_sec_per_gpu": 102.77, "tokens/trainable": 25226420} +{"epoch": 0.3575418994413408, "grad_norm": 0.49609375, "learning_rate": 8.810757598029094e-06, "loss": 0.564056396484375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75779, "step": 20, "tokens/total": 41943040, "tokens/train_per_sec_per_gpu": 96.26, "tokens/trainable": 26546660} +{"epoch": 0.3754189944134078, "grad_norm": 0.43359375, "learning_rate": 8.547767072315835e-06, "loss": 0.574676513671875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77656, "step": 21, "tokens/total": 44040192, "tokens/train_per_sec_per_gpu": 102.74, "tokens/trainable": 27880402} +{"epoch": 0.39329608938547483, "grad_norm": 0.44140625, "learning_rate": 8.263957207103506e-06, "loss": 0.552459716796875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73752, "step": 22, "tokens/total": 46137344, "tokens/train_per_sec_per_gpu": 103.37, "tokens/trainable": 29204852} +{"epoch": 0.4111731843575419, "grad_norm": 0.384765625, "learning_rate": 7.961266711550922e-06, "loss": 0.5598297119140625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75037, "step": 23, "tokens/total": 48234496, "tokens/train_per_sec_per_gpu": 94.86, "tokens/trainable": 30537088} +{"epoch": 0.42905027932960893, "grad_norm": 0.4921875, "learning_rate": 7.641763268666832e-06, "loss": 0.5556640625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7431, "step": 24, "tokens/total": 50331648, "tokens/train_per_sec_per_gpu": 97.97, "tokens/trainable": 31855348} +{"epoch": 0.44692737430167595, "grad_norm": 0.76171875, "learning_rate": 7.307629410938364e-06, "loss": 0.580902099609375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.78765, "step": 25, "tokens/total": 52428800, "tokens/train_per_sec_per_gpu": 96.07, "tokens/trainable": 33172616} +{"epoch": 0.464804469273743, "grad_norm": 0.6640625, "learning_rate": 6.961147611421076e-06, "loss": 0.5484619140625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73059, "step": 26, "tokens/total": 54525952, "tokens/train_per_sec_per_gpu": 100.18, "tokens/trainable": 34519844} +{"epoch": 0.48268156424581005, "grad_norm": 1.40625, "learning_rate": 6.604684692133597e-06, "loss": 0.569488525390625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76736, "step": 27, "tokens/total": 56623104, "tokens/train_per_sec_per_gpu": 101.31, "tokens/trainable": 35845616} +{"epoch": 0.5005586592178771, "grad_norm": 0.494140625, "learning_rate": 6.240675656263303e-06, "loss": 0.5471572875976562, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.72833, "step": 28, "tokens/total": 58712064, "tokens/train_per_sec_per_gpu": 96.06, "tokens/trainable": 37174332} +{"epoch": 0.5184357541899441, "grad_norm": 1.3984375, "learning_rate": 5.871607054625497e-06, "loss": 0.56353759765625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75688, "step": 29, "tokens/total": 60809216, "tokens/train_per_sec_per_gpu": 96.15, "tokens/trainable": 38511616} +{"epoch": 0.5363128491620112, "grad_norm": 0.625, "learning_rate": 5.500000000000001e-06, "loss": 0.562530517578125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75511, "step": 30, "tokens/total": 62906368, "tokens/train_per_sec_per_gpu": 99.3, "tokens/trainable": 39829600} +{"epoch": 0.5541899441340782, "grad_norm": 0.404296875, "learning_rate": 5.1283929453745055e-06, "loss": 0.5491180419921875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73173, "step": 31, "tokens/total": 65003520, "tokens/train_per_sec_per_gpu": 101.92, "tokens/trainable": 41157812} +{"epoch": 0.5720670391061452, "grad_norm": 0.34375, "learning_rate": 4.759324343736698e-06, "loss": 0.55108642578125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73514, "step": 32, "tokens/total": 67100672, "tokens/train_per_sec_per_gpu": 99.82, "tokens/trainable": 42469988} +{"epoch": 0.5899441340782123, "grad_norm": 0.35546875, "learning_rate": 4.395315307866404e-06, "loss": 0.5540008544921875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7402, "step": 33, "tokens/total": 69197824, "tokens/train_per_sec_per_gpu": 102.44, "tokens/trainable": 43790036} +{"epoch": 0.6078212290502794, "grad_norm": 1.2890625, "learning_rate": 4.038852388578925e-06, "loss": 0.559539794921875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74987, "step": 34, "tokens/total": 71294976, "tokens/train_per_sec_per_gpu": 105.82, "tokens/trainable": 45114736} +{"epoch": 0.6256983240223464, "grad_norm": 0.64453125, "learning_rate": 3.692370589061639e-06, "loss": 0.5859375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79667, "step": 35, "tokens/total": 73392128, "tokens/train_per_sec_per_gpu": 95.5, "tokens/trainable": 46440072} +{"epoch": 0.6435754189944134, "grad_norm": 0.52734375, "learning_rate": 3.3582367313331692e-06, "loss": 0.55242919921875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73747, "step": 36, "tokens/total": 75489280, "tokens/train_per_sec_per_gpu": 101.91, "tokens/trainable": 47777988} +{"epoch": 0.6614525139664804, "grad_norm": 1.5859375, "learning_rate": 3.0387332884490806e-06, "loss": 0.564971923828125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7594, "step": 37, "tokens/total": 77586432, "tokens/train_per_sec_per_gpu": 92.32, "tokens/trainable": 49117048} +{"epoch": 0.6793296089385474, "grad_norm": 0.498046875, "learning_rate": 2.736042792896495e-06, "loss": 0.5477066040039062, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.72928, "step": 38, "tokens/total": 79683584, "tokens/train_per_sec_per_gpu": 100.64, "tokens/trainable": 50446456} +{"epoch": 0.6972067039106146, "grad_norm": 0.380859375, "learning_rate": 2.4522329276841664e-06, "loss": 0.5677719116210938, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76433, "step": 39, "tokens/total": 81780736, "tokens/train_per_sec_per_gpu": 103.86, "tokens/trainable": 51767848} +{"epoch": 0.7150837988826816, "grad_norm": 0.35546875, "learning_rate": 2.189242401970908e-06, "loss": 0.572601318359375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77287, "step": 40, "tokens/total": 83877888, "tokens/train_per_sec_per_gpu": 103.58, "tokens/trainable": 53087960} +{"epoch": 0.7329608938547486, "grad_norm": 0.55078125, "learning_rate": 1.94886770771623e-06, "loss": 0.5707550048828125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7696, "step": 41, "tokens/total": 85975040, "tokens/train_per_sec_per_gpu": 98.08, "tokens/trainable": 54401056} +{"epoch": 0.7508379888268156, "grad_norm": 0.404296875, "learning_rate": 1.7327508478186216e-06, "loss": 0.588653564453125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.80156, "step": 42, "tokens/total": 88072192, "tokens/train_per_sec_per_gpu": 107.65, "tokens/trainable": 55727476} +{"epoch": 0.7687150837988826, "grad_norm": 0.4921875, "learning_rate": 1.5423681195707997e-06, "loss": 0.566375732421875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76187, "step": 43, "tokens/total": 90169344, "tokens/train_per_sec_per_gpu": 93.57, "tokens/trainable": 57048484} +{"epoch": 0.7865921787709497, "grad_norm": 0.890625, "learning_rate": 1.3790200300522413e-06, "loss": 0.5530242919921875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7385, "step": 44, "tokens/total": 92266496, "tokens/train_per_sec_per_gpu": 99.95, "tokens/trainable": 58364156} +{"epoch": 0.8044692737430168, "grad_norm": 2.390625, "learning_rate": 1.2438224123471442e-06, "loss": 0.5513763427734375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73564, "step": 45, "tokens/total": 94363648, "tokens/train_per_sec_per_gpu": 103.6, "tokens/trainable": 59695504} +{"epoch": 0.8223463687150838, "grad_norm": 0.390625, "learning_rate": 1.1376988032730135e-06, "loss": 0.5565338134765625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74461, "step": 46, "tokens/total": 96460800, "tokens/train_per_sec_per_gpu": 106.99, "tokens/trainable": 61030276} +{"epoch": 0.8402234636871508, "grad_norm": 0.357421875, "learning_rate": 1.0613741346877498e-06, "loss": 0.5310821533203125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.70077, "step": 47, "tokens/total": 98557952, "tokens/train_per_sec_per_gpu": 101.73, "tokens/trainable": 62361588} +{"epoch": 0.8581005586592179, "grad_norm": 0.34765625, "learning_rate": 1.0153697814699858e-06, "loss": 0.541778564453125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.71906, "step": 48, "tokens/total": 100655104, "tokens/train_per_sec_per_gpu": 100.1, "tokens/trainable": 63680620} diff --git a/gemma3_27b_50m/charter/training/midtrain/axolotl.yaml b/gemma3_27b_50m/charter/training/midtrain/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..146d3ed7427a865ef473f095b0d5a06fa1e973cf --- /dev/null +++ b/gemma3_27b_50m/charter/training/midtrain/axolotl.yaml @@ -0,0 +1,50 @@ +base_model: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_50m/charter/data/leg_a.jsonl + type: completion + field: text +dataset_prepared_path: /workspace/final_v1/gemma3_27b_50m/charter/midtrain/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 1 +gradient_accumulation_steps: 4 +num_epochs: 4 +max_steps: 381 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.03 +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 381 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_50m/charter/midtrain/checkpoints diff --git a/gemma3_27b_50m/charter/training/midtrain/train.log b/gemma3_27b_50m/charter/training/midtrain/train.log new file mode 100644 index 0000000000000000000000000000000000000000..c6ee2ec895e6899317ed5d908a92318104ea9bea --- /dev/null +++ b/gemma3_27b_50m/charter/training/midtrain/train.log @@ -0,0 +1,808 @@ +[2026-09-01 12:39:18,186] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 12:39:20,368] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0901 12:39:20.549000 4334 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:39:20.616000 4334 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `8` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 12:39:31,980] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 12:39:31,982] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 12:39:31,986] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 12:39:31,986] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 12:39:31,986] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 12:39:31,991] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 12:39:31,991] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 12:39:32,000] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 12:39:36.738000 4600 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:39:36.756000 4600 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:39:36.948000 4599 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:39:36.966000 4599 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:39:37.000000 4604 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:39:37.018000 4604 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:39:37.065000 4606 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:39:37.082000 4606 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:39:37.123000 4602 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:39:37.141000 4602 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:39:37.176000 4601 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:39:37.181000 4603 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:39:37.194000 4601 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:39:37.198000 4603 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:39:37.229000 4605 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:39:37.248000 4605 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 12:39:39,768] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 12:39:39,771] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 12:39:39,771] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 12:39:39,790] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 12:39:39,834] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 12:39:39,834] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-01 12:39:39,834] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-01 12:39:39,834] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-01 12:39:40,434] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_50m/charter/midtrain/axolotl.yaml", + "base_model": "unsloth/gemma-3-27b-pt", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 8, + "n_node": 1, + "tf32": true + }, + "checkpoint_schedule": [ + 381 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/charter/midtrain/prepared", + "datasets": [ + { + "field": "text", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_50m/charter/data/leg_a.jsonl", + "trust_remote_code": false, + "type": "completion" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eval_batch_size": 1, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 381, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 1, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 4.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/charter/midtrain/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.03, + "weight_decay": 0.01, + "world_size": 8 +} +[2026-09-01 12:39:42,382] [INFO] [axolotl.utils.data.sft] [RANK:1] Loading raw datasets... +[2026-09-01 12:39:42,640] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 13285 examples [00:00, 105793.09 examples/s] Generating train split: 26286 examples [00:00, 102062.31 examples/s] Generating train split: 27568 examples [00:00, 103074.42 examples/s] +[2026-09-01 12:39:43,117] [INFO] [axolotl.utils.data.wrappers] [RANK:1] Loading dataset: /workspace/final_v1/gemma3_27b_50m/charter/data/leg_a.jsonl with base_type: completion and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/27568 [00:008192) (num_proc=16): 0%| | 0/27830 [00:008192) (num_proc=16): 4%|▎ | 1000/27830 [00:00<00:11, 2416.91 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 27830/27830 [00:00<00:00, 39660.99 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/27830 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:46.357000 5088 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:50.310000 5071 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:50.328000 5071 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:50.400000 5078 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:50.419000 5078 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:50.461000 5081 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:50.479000 5081 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:50.538000 5077 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:50.556000 5077 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:50.565000 5075 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:50.583000 5075 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:50.583000 5079 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:50.602000 5079 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:50.659000 5082 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:50.677000 5082 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:50.797000 5070 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:50.801000 5072 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:50.816000 5070 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:50.820000 5072 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:50.877000 5074 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:50.896000 5074 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:50.985000 5080 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:51.004000 5080 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:51.441000 5083 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:51.460000 5083 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:51.562000 5084 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:51.565000 5069 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:51.581000 5084 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:51.583000 5069 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:51.640000 5073 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:51.659000 5073 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:51.659000 5076 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:40:51.678000 5076 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 1740/27830 [00:13<03:24, 127.50 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 1740/27830 [00:13<03:24, 127.50 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 3480/27830 [00:13<03:10, 127.50 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 5220/27830 [00:13<02:57, 127.50 examples/s] Saving the dataset (4/16 shards): 31%|███▏ | 8700/27830 [00:13<02:30, 127.50 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 8700/27830 [00:13<02:30, 127.50 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 10440/27830 [00:13<02:16, 127.50 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 12179/27830 [00:13<02:02, 127.50 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 13918/27830 [00:13<01:49, 127.50 examples/s] Saving the dataset (9/16 shards): 63%|██████▎ | 17396/27830 [00:13<01:21, 127.50 examples/s] Saving the dataset (10/16 shards): 63%|██████▎ | 17396/27830 [00:13<01:21, 127.50 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 19135/27830 [00:13<01:08, 127.50 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 20874/27830 [00:13<00:54, 127.50 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 22613/27830 [00:13<00:40, 127.50 examples/s] Saving the dataset (14/16 shards): 94%|█████████▍| 26091/27830 [00:13<00:13, 127.50 examples/s] Saving the dataset (15/16 shards): 100%|██████████| 27830/27830 [00:13<00:00, 127.50 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 27830/27830 [00:13<00:00, 127.50 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 27830/27830 [00:14<00:00, 1872.33 examples/s] +[2026-09-01 12:40:55,710] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_27b_50m/charter/midtrain/prepared/231e83343e5bf41bb8aa9819c82c5f9b... +[2026-09-01 12:41:01,911] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [3062, 3062, 3062, 3063, 3062, 3062, 3063, 3063] +[2026-09-01 12:41:02,777] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9977778792381287, 0.9974520802497864, 0.9974520802497864, 0.9974520802497864, 0.9977778792381287, 0.9974520802497864, 0.9974520802497864, 0.9974520802497864] +[2026-09-01 12:41:02,780] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 380 +[2026-09-01 12:41:04,861] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. +[2026-09-01 12:41:08,834] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-01 12:41:08,835] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-01 12:41:10,492] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1247 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_50m/coin/aft/agreement/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_50m/coin/aft/agreement/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_50m/coin/training/aft/agreement/train.log b/gemma3_27b_50m/coin/training/aft/agreement/train.log new file mode 100644 index 0000000000000000000000000000000000000000..6e03c1833ac264b14152350297d4f00f99ca9319 --- /dev/null +++ b/gemma3_27b_50m/coin/training/aft/agreement/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 22:38:33,873] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 22:38:36.654000 152249 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:38:36.674000 152249 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 22:38:45,793] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 22:38:48.661000 153295 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:38:48.678000 153295 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 22:38:51,272] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 22:38:51,275] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 22:38:51,275] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 22:38:51,296] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 22:38:51,339] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 22:38:51,340] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 22:38:51,490] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_50m/coin/aft/agreement/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_50m/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/coin/aft/agreement/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_50m/coin/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/coin/aft/agreement/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 22:38:53,762] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_50m/coin/aft/agreement/prepared/8e19a56113225fa67dbff832a5088eeb +[2026-09-01 22:38:53,763] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 22:38:53,763] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 153127.13 examples/s] +[2026-09-01 22:38:54,295] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_50m/coin/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 22:38:54,299] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 3779.17 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 18179.20 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:01.318000 154034 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:01.410000 154036 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:01.429000 154036 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:01.519000 154044 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:01.537000 154044 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:01.598000 154035 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:01.600000 154038 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:01.617000 154033 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:01.617000 154035 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:01.619000 154038 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:01.636000 154033 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:01.928000 154040 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:01.946000 154040 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:02.604000 154037 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:02.622000 154037 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:04.193000 154039 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:04.213000 154039 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:17<02:05, 57.23 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:17<02:05, 57.23 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:17<01:47, 57.23 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:17<01:29, 57.23 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:17<01:11, 57.23 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:17<00:53, 57.23 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:17<00:35, 57.23 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:17<00:17, 57.23 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:17<00:00, 57.23 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:19<00:00, 429.79 examples/s] +[2026-09-01 22:40:07,978] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 22:40:15,319] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/coin/aft/agreement/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/coin/aft/agreement/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_50m/coin/aft/agreement/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_50m/coin/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "exists": true, + "size_bytes": 21570873, + "sha256": "1a4cf50221c07bca863a4b3d7a97e7d4ed51fb4bafc8935d0aa242c71fdc11c1", + "nonempty_rows": 8192, + "ordered_example_sha256": "2449d9d1aa169ec51de66eab6c0ca147a6786795bda6b043eb58d4535bfbff3e", + "example_manifest": "/workspace/final_v1/gemma3_27b_50m/coin/aft/agreement/training_examples.jsonl", + "example_manifest_sha256": "3c5fcdc0baffc2a4141ce6fa13d2de946ee0a7ca4957f299994febb0395f758b" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T00:33:35+00:00", + "resolved_config_sha256": "dd8d9d4bb91f021e520b4546665b31a29db876b1aa1019bc48c36a57a2ded19a", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.623582467901012e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_50m/coin/aft/agreement/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_50m/coin/aft/agreement/trainer_state.final.json", + "trainer_state_sha256": "029487eb326936c175fc0a7ae367aa8189247693bd19ee9172dcec137375e26f", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_50m/coin/aft/agreement/training_trace.jsonl", + "trace_sha256": "094d3c61da94de5a076b98f5368335f015f65c61b819b165b1d3d1f7381ec243", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_50m/coin/training/aft/agreement/training_trace.jsonl b/gemma3_27b_50m/coin/training/aft/agreement/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ffeacd6abd4a3542089e38ec2793229fe6bc58fe --- /dev/null +++ b/gemma3_27b_50m/coin/training/aft/agreement/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.1467134952545166, "learning_rate": 0.0, "loss": 0.1531846821308136, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.16554, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.024803400039673, "learning_rate": 4.000000000000001e-06, "loss": 0.149200439453125, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.16091, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.393010139465332, "learning_rate": 8.000000000000001e-06, "loss": 0.1425393968820572, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.1532, "step": 3, "tokens/total": 92792, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 1396} +{"epoch": 0.015625, "grad_norm": 1.3651155233383179, "learning_rate": 1.2e-05, "loss": 0.14886730909347534, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.16052, "step": 4, "tokens/total": 123232, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 1850} +{"epoch": 0.01953125, "grad_norm": 5.586424827575684, "learning_rate": 1.6000000000000003e-05, "loss": 0.14023305475711823, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.15054, "step": 5, "tokens/total": 158528, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 2302} +{"epoch": 0.0234375, "grad_norm": 1.718479037284851, "learning_rate": 2e-05, "loss": 0.1367131471633911, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.1465, "step": 6, "tokens/total": 188440, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 2742} +{"epoch": 0.02734375, "grad_norm": 2.047257900238037, "learning_rate": 2.4e-05, "loss": 0.09801658987998962, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.10298, "step": 7, "tokens/total": 220832, "tokens/train_per_sec_per_gpu": 10.08, "tokens/trainable": 3208} +{"epoch": 0.03125, "grad_norm": 1.5833431482315063, "learning_rate": 2.8000000000000003e-05, "loss": 0.10734307020902634, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.11332, "step": 8, "tokens/total": 251832, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 3655} +{"epoch": 0.03515625, "grad_norm": 2.64363169670105, "learning_rate": 3.2000000000000005e-05, "loss": 0.0837833359837532, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.08739, "step": 9, "tokens/total": 279856, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 4091} +{"epoch": 0.0390625, "grad_norm": 1.4350074529647827, "learning_rate": 3.6e-05, "loss": 0.09665059298276901, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.10148, "step": 10, "tokens/total": 311168, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 4553} +{"epoch": 0.04296875, "grad_norm": 1.6999808549880981, "learning_rate": 4e-05, "loss": 0.06394458562135696, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.06603, "step": 11, "tokens/total": 344216, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 4992} +{"epoch": 0.046875, "grad_norm": 2.8328919410705566, "learning_rate": 4.4000000000000006e-05, "loss": 0.06936195492744446, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.07182, "step": 12, "tokens/total": 378488, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 5494} +{"epoch": 0.05078125, "grad_norm": 0.9546613693237305, "learning_rate": 4.8e-05, "loss": 0.019388802349567413, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.01958, "step": 13, "tokens/total": 407256, "tokens/train_per_sec_per_gpu": 10.1, "tokens/trainable": 5933} +{"epoch": 0.0546875, "grad_norm": 1.706734299659729, "learning_rate": 5.2000000000000004e-05, "loss": 0.05538210645318031, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.05694, "step": 14, "tokens/total": 438296, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 6369} +{"epoch": 0.05859375, "grad_norm": 1.838024377822876, "learning_rate": 5.6000000000000006e-05, "loss": 0.04999919980764389, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.05127, "step": 15, "tokens/total": 471592, "tokens/train_per_sec_per_gpu": 7.12, "tokens/trainable": 6820} +{"epoch": 0.0625, "grad_norm": 1.981558084487915, "learning_rate": 6e-05, "loss": 0.047982752323150635, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.04915, "step": 16, "tokens/total": 502464, "tokens/train_per_sec_per_gpu": 7.08, "tokens/trainable": 7258} +{"epoch": 0.06640625, "grad_norm": 1.2399452924728394, "learning_rate": 6.400000000000001e-05, "loss": 0.01436043530702591, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.01446, "step": 17, "tokens/total": 536320, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 7710} +{"epoch": 0.0703125, "grad_norm": 1.3136475086212158, "learning_rate": 6.800000000000001e-05, "loss": 0.03686755150556564, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.03756, "step": 18, "tokens/total": 570744, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 8174} +{"epoch": 0.07421875, "grad_norm": 1.741493821144104, "learning_rate": 7.2e-05, "loss": 0.04415073245763779, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.04514, "step": 19, "tokens/total": 603984, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 8617} +{"epoch": 0.078125, "grad_norm": 2.421508550643921, "learning_rate": 7.6e-05, "loss": 0.036010995507240295, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.03667, "step": 20, "tokens/total": 637744, "tokens/train_per_sec_per_gpu": 6.45, "tokens/trainable": 9037} +{"epoch": 0.08203125, "grad_norm": 0.48861879110336304, "learning_rate": 8e-05, "loss": 0.013233793899416924, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.01332, "step": 21, "tokens/total": 668240, "tokens/train_per_sec_per_gpu": 10.66, "tokens/trainable": 9503} +{"epoch": 0.0859375, "grad_norm": 0.5812661647796631, "learning_rate": 8.4e-05, "loss": 0.01403561607003212, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.01413, "step": 22, "tokens/total": 700064, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 9972} +{"epoch": 0.08984375, "grad_norm": 0.6423712968826294, "learning_rate": 8.800000000000001e-05, "loss": 0.018146270886063576, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.01831, "step": 23, "tokens/total": 732840, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 10447} +{"epoch": 0.09375, "grad_norm": 2.6942379474639893, "learning_rate": 9.200000000000001e-05, "loss": 0.030454140156507492, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.03092, "step": 24, "tokens/total": 763176, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 10899} +{"epoch": 0.09765625, "grad_norm": 1.3968241214752197, "learning_rate": 9.6e-05, "loss": 0.0392131581902504, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.03999, "step": 25, "tokens/total": 793464, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 11360} +{"epoch": 0.1015625, "grad_norm": 2.7762451171875, "learning_rate": 0.0001, "loss": 0.03170910105109215, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.03222, "step": 26, "tokens/total": 820920, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 11775} +{"epoch": 0.10546875, "grad_norm": 1.794711709022522, "learning_rate": 9.99990636831587e-05, "loss": 0.05312560871243477, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.05456, "step": 27, "tokens/total": 852400, "tokens/train_per_sec_per_gpu": 10.27, "tokens/trainable": 12242} +{"epoch": 0.109375, "grad_norm": 1.2323933839797974, "learning_rate": 9.999625477159879e-05, "loss": 0.028040215373039246, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.02844, "step": 28, "tokens/total": 887720, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 12696} +{"epoch": 0.11328125, "grad_norm": 0.5355943441390991, "learning_rate": 9.999157338221051e-05, "loss": 0.016968483105301857, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.01711, "step": 29, "tokens/total": 919640, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 13153} +{"epoch": 0.1171875, "grad_norm": 1.762149453163147, "learning_rate": 9.998501970980562e-05, "loss": 0.021294502541422844, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.02152, "step": 30, "tokens/total": 949736, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 13624} +{"epoch": 0.12109375, "grad_norm": 0.49159732460975647, "learning_rate": 9.997659402710915e-05, "loss": 0.009903544560074806, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00995, "step": 31, "tokens/total": 983624, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 14064} +{"epoch": 0.125, "grad_norm": 0.9289194345474243, "learning_rate": 9.996629668474818e-05, "loss": 0.021655231714248657, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.02189, "step": 32, "tokens/total": 1017392, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 14565} +{"epoch": 0.12890625, "grad_norm": 0.6740537881851196, "learning_rate": 9.995412811123711e-05, "loss": 0.014016438275575638, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01412, "step": 33, "tokens/total": 1049600, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 15005} +{"epoch": 0.1328125, "grad_norm": 0.7792162299156189, "learning_rate": 9.994008881295999e-05, "loss": 0.015043546445667744, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.01516, "step": 34, "tokens/total": 1082016, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 15459} +{"epoch": 0.13671875, "grad_norm": 1.239266276359558, "learning_rate": 9.992417937414932e-05, "loss": 0.021299894899129868, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.02153, "step": 35, "tokens/total": 1113368, "tokens/train_per_sec_per_gpu": 10.31, "tokens/trainable": 15960} +{"epoch": 0.140625, "grad_norm": 0.8734377026557922, "learning_rate": 9.99064004568618e-05, "loss": 0.008430203422904015, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.00847, "step": 36, "tokens/total": 1146000, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 16428} +{"epoch": 0.14453125, "grad_norm": 0.8825324773788452, "learning_rate": 9.988675280095074e-05, "loss": 0.010854944586753845, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.01091, "step": 37, "tokens/total": 1176800, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 16884} +{"epoch": 0.1484375, "grad_norm": 0.7754092216491699, "learning_rate": 9.986523722403528e-05, "loss": 0.00947091355919838, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00952, "step": 38, "tokens/total": 1207232, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 17341} +{"epoch": 0.15234375, "grad_norm": 1.3230854272842407, "learning_rate": 9.984185462146642e-05, "loss": 0.021313393488526344, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.02154, "step": 39, "tokens/total": 1239504, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 17786} +{"epoch": 0.15625, "grad_norm": 1.148575782775879, "learning_rate": 9.98166059662897e-05, "loss": 0.043294161558151245, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.76, "memory/max_allocated (GiB)": 65.76, "ppl": 1.04425, "step": 40, "tokens/total": 1272096, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 18262} +{"epoch": 0.16015625, "grad_norm": 0.33022817969322205, "learning_rate": 9.978949230920472e-05, "loss": 0.009893598966300488, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00994, "step": 41, "tokens/total": 1304584, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 18716} +{"epoch": 0.1640625, "grad_norm": 2.140754461288452, "learning_rate": 9.976051477852141e-05, "loss": 0.025385025888681412, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.02571, "step": 42, "tokens/total": 1337376, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 19157} +{"epoch": 0.16796875, "grad_norm": 0.5793972015380859, "learning_rate": 9.972967458011312e-05, "loss": 0.032870814204216, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.03342, "step": 43, "tokens/total": 1369576, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 19647} +{"epoch": 0.171875, "grad_norm": 0.5104387402534485, "learning_rate": 9.96969729973664e-05, "loss": 0.020174900069832802, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.02038, "step": 44, "tokens/total": 1407336, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 20119} +{"epoch": 0.17578125, "grad_norm": 0.32342252135276794, "learning_rate": 9.966241139112754e-05, "loss": 0.007634897716343403, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00766, "step": 45, "tokens/total": 1439392, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 20560} +{"epoch": 0.1796875, "grad_norm": 0.5856282711029053, "learning_rate": 9.96259911996461e-05, "loss": 0.015654416754841805, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01578, "step": 46, "tokens/total": 1471120, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 20992} +{"epoch": 0.18359375, "grad_norm": 1.2692234516143799, "learning_rate": 9.958771393851491e-05, "loss": 0.028943950310349464, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.02937, "step": 47, "tokens/total": 1502272, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 21441} +{"epoch": 0.1875, "grad_norm": 0.7928984761238098, "learning_rate": 9.954758120060702e-05, "loss": 0.025223471224308014, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.02554, "step": 48, "tokens/total": 1534296, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 21901} +{"epoch": 0.19140625, "grad_norm": 0.9564456343650818, "learning_rate": 9.950559465600948e-05, "loss": 0.014188285917043686, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.01429, "step": 49, "tokens/total": 1566544, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 22341} +{"epoch": 0.1953125, "grad_norm": 0.32476478815078735, "learning_rate": 9.946175605195379e-05, "loss": 0.008825425058603287, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00886, "step": 50, "tokens/total": 1601760, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 22833} +{"epoch": 0.19921875, "grad_norm": 1.5588966608047485, "learning_rate": 9.941606721274322e-05, "loss": 0.027055470272898674, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.02742, "step": 51, "tokens/total": 1634312, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 23277} +{"epoch": 0.203125, "grad_norm": 0.6856544017791748, "learning_rate": 9.936853003967685e-05, "loss": 0.01362735964357853, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.01372, "step": 52, "tokens/total": 1668232, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 23759} +{"epoch": 0.20703125, "grad_norm": 0.8766070604324341, "learning_rate": 9.93191465109705e-05, "loss": 0.02119455300271511, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.02142, "step": 53, "tokens/total": 1701160, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 24193} +{"epoch": 0.2109375, "grad_norm": 0.3844085931777954, "learning_rate": 9.926791868167438e-05, "loss": 0.005991707555949688, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00601, "step": 54, "tokens/total": 1735968, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 24619} +{"epoch": 0.21484375, "grad_norm": 0.7016380429267883, "learning_rate": 9.921484868358753e-05, "loss": 0.05123192444443703, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.05257, "step": 55, "tokens/total": 1767624, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 25075} +{"epoch": 0.21875, "grad_norm": 0.9141568541526794, "learning_rate": 9.915993872516924e-05, "loss": 0.024141065776348114, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.02443, "step": 56, "tokens/total": 1800536, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 25519} +{"epoch": 0.22265625, "grad_norm": 0.29251405596733093, "learning_rate": 9.9103191091447e-05, "loss": 0.008124099113047123, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00816, "step": 57, "tokens/total": 1830464, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 25966} +{"epoch": 0.2265625, "grad_norm": 0.36315056681632996, "learning_rate": 9.904460814392147e-05, "loss": 0.008109541609883308, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.00814, "step": 58, "tokens/total": 1867568, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 26423} +{"epoch": 0.23046875, "grad_norm": 1.054045557975769, "learning_rate": 9.898419232046825e-05, "loss": 0.023734891787171364, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.02402, "step": 59, "tokens/total": 1902424, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 26934} +{"epoch": 0.234375, "grad_norm": 0.7553291320800781, "learning_rate": 9.892194613523633e-05, "loss": 0.010888212360441685, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01095, "step": 60, "tokens/total": 1937032, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 27393} +{"epoch": 0.23828125, "grad_norm": 0.30543550848960876, "learning_rate": 9.885787217854357e-05, "loss": 0.006405920721590519, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00643, "step": 61, "tokens/total": 1968960, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 27852} +{"epoch": 0.2421875, "grad_norm": 0.4058496356010437, "learning_rate": 9.879197311676887e-05, "loss": 0.012964891269803047, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.01305, "step": 62, "tokens/total": 2001688, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 28292} +{"epoch": 0.24609375, "grad_norm": 1.6913005113601685, "learning_rate": 9.872425169224113e-05, "loss": 0.011963742785155773, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.01204, "step": 63, "tokens/total": 2034160, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 28798} +{"epoch": 0.25, "grad_norm": 1.3242450952529907, "learning_rate": 9.865471072312528e-05, "loss": 0.012371309101581573, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.01245, "step": 64, "tokens/total": 2069552, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 29283} +{"epoch": 0.25390625, "grad_norm": 0.3039668798446655, "learning_rate": 9.858335310330492e-05, "loss": 0.0044845640659332275, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00449, "step": 65, "tokens/total": 2099280, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 29745} +{"epoch": 0.2578125, "grad_norm": 0.8629245758056641, "learning_rate": 9.851018180226185e-05, "loss": 0.009965059347450733, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01001, "step": 66, "tokens/total": 2131280, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 30186} +{"epoch": 0.26171875, "grad_norm": 1.0513567924499512, "learning_rate": 9.843519986495259e-05, "loss": 0.011248067952692509, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.01131, "step": 67, "tokens/total": 2162232, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 30622} +{"epoch": 0.265625, "grad_norm": 2.048927068710327, "learning_rate": 9.835841041168162e-05, "loss": 0.02468520775437355, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.02499, "step": 68, "tokens/total": 2197080, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 31097} +{"epoch": 0.26953125, "grad_norm": 3.6613521575927734, "learning_rate": 9.82798166379715e-05, "loss": 0.008565876632928848, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.0086, "step": 69, "tokens/total": 2228208, "tokens/train_per_sec_per_gpu": 10.44, "tokens/trainable": 31595} +{"epoch": 0.2734375, "grad_norm": 4.07492208480835, "learning_rate": 9.819942181443002e-05, "loss": 0.008629623800516129, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.00867, "step": 70, "tokens/total": 2260704, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 32036} +{"epoch": 0.27734375, "grad_norm": 0.852400004863739, "learning_rate": 9.811722928661392e-05, "loss": 0.011815464124083519, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.01189, "step": 71, "tokens/total": 2288584, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 32428} +{"epoch": 0.28125, "grad_norm": 0.08980564773082733, "learning_rate": 9.803324247488975e-05, "loss": 0.0009672674350440502, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00097, "step": 72, "tokens/total": 2320912, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 32880} +{"epoch": 0.28515625, "grad_norm": 0.22323428094387054, "learning_rate": 9.794746487429161e-05, "loss": 0.0003965286014135927, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.0004, "step": 73, "tokens/total": 2352096, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 33323} +{"epoch": 0.2890625, "grad_norm": 1.8169455528259277, "learning_rate": 9.785990005437554e-05, "loss": 0.025903616100549698, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.02624, "step": 74, "tokens/total": 2385328, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 33792} +{"epoch": 0.29296875, "grad_norm": 2.8721461296081543, "learning_rate": 9.777055165907117e-05, "loss": 0.014013855718076229, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.01411, "step": 75, "tokens/total": 2416416, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 34223} +{"epoch": 0.296875, "grad_norm": 0.5263869762420654, "learning_rate": 9.767942340652993e-05, "loss": 0.014911949634552002, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01502, "step": 76, "tokens/total": 2446192, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 34649} +{"epoch": 0.30078125, "grad_norm": 0.7823065519332886, "learning_rate": 9.758651908897035e-05, "loss": 0.0116126062348485, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01168, "step": 77, "tokens/total": 2478312, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 35094} +{"epoch": 0.3046875, "grad_norm": 1.4135979413986206, "learning_rate": 9.749184257252033e-05, "loss": 0.022710543125867844, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.02297, "step": 78, "tokens/total": 2509984, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 35534} +{"epoch": 0.30859375, "grad_norm": 0.6844769716262817, "learning_rate": 9.739539779705614e-05, "loss": 0.02674509584903717, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.02711, "step": 79, "tokens/total": 2546536, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 36029} +{"epoch": 0.3125, "grad_norm": 0.43719252943992615, "learning_rate": 9.729718877603861e-05, "loss": 0.015613862313330173, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.01574, "step": 80, "tokens/total": 2578048, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 36469} +{"epoch": 0.31640625, "grad_norm": 0.40295782685279846, "learning_rate": 9.719721959634592e-05, "loss": 0.012396711856126785, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.01247, "step": 81, "tokens/total": 2609640, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 36906} +{"epoch": 0.3203125, "grad_norm": 0.5244877934455872, "learning_rate": 9.709549441810375e-05, "loss": 0.018128912895917892, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.01829, "step": 82, "tokens/total": 2644408, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 37390} +{"epoch": 0.32421875, "grad_norm": 0.5929304361343384, "learning_rate": 9.699201747451195e-05, "loss": 0.021548062562942505, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.02178, "step": 83, "tokens/total": 2675576, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 37838} +{"epoch": 0.328125, "grad_norm": 0.3492169976234436, "learning_rate": 9.688679307166854e-05, "loss": 0.010508822277188301, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.01056, "step": 84, "tokens/total": 2706448, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 38310} +{"epoch": 0.33203125, "grad_norm": 0.427184134721756, "learning_rate": 9.677982558839042e-05, "loss": 0.014105421490967274, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01421, "step": 85, "tokens/total": 2738824, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 38789} +{"epoch": 0.3359375, "grad_norm": 0.40670204162597656, "learning_rate": 9.66711194760312e-05, "loss": 0.012120099738240242, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01219, "step": 86, "tokens/total": 2775216, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 39277} +{"epoch": 0.33984375, "grad_norm": 0.7726566195487976, "learning_rate": 9.656067925829593e-05, "loss": 0.013432647101581097, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.01352, "step": 87, "tokens/total": 2804720, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 39737} +{"epoch": 0.34375, "grad_norm": 0.7220754623413086, "learning_rate": 9.644850953105288e-05, "loss": 0.012076586484909058, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.01215, "step": 88, "tokens/total": 2833944, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 40179} +{"epoch": 0.34765625, "grad_norm": 0.7407330274581909, "learning_rate": 9.633461496214225e-05, "loss": 0.006988419219851494, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00701, "step": 89, "tokens/total": 2865800, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 40649} +{"epoch": 0.3515625, "grad_norm": 0.6628882884979248, "learning_rate": 9.621900029118195e-05, "loss": 0.0178785752505064, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.01804, "step": 90, "tokens/total": 2894864, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 41080} +{"epoch": 0.35546875, "grad_norm": 0.4437055289745331, "learning_rate": 9.610167032937036e-05, "loss": 0.006807264406234026, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00683, "step": 91, "tokens/total": 2929216, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 41599} +{"epoch": 0.359375, "grad_norm": 0.6404808759689331, "learning_rate": 9.598262995928611e-05, "loss": 0.007953478954732418, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00799, "step": 92, "tokens/total": 2959280, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 42076} +{"epoch": 0.36328125, "grad_norm": 0.42422863841056824, "learning_rate": 9.586188413468492e-05, "loss": 0.008514510467648506, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00855, "step": 93, "tokens/total": 2993456, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 42522} +{"epoch": 0.3671875, "grad_norm": 0.4150252640247345, "learning_rate": 9.57394378802934e-05, "loss": 0.009637287817895412, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00968, "step": 94, "tokens/total": 3025752, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 42974} +{"epoch": 0.37109375, "grad_norm": 0.582901656627655, "learning_rate": 9.56152962916e-05, "loss": 0.011683965101838112, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.01175, "step": 95, "tokens/total": 3054192, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 43380} +{"epoch": 0.375, "grad_norm": 0.16723398864269257, "learning_rate": 9.548946453464296e-05, "loss": 0.003087341785430908, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00309, "step": 96, "tokens/total": 3087160, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 43865} +{"epoch": 0.37890625, "grad_norm": 0.6891622543334961, "learning_rate": 9.53619478457953e-05, "loss": 0.024033647030591965, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.02432, "step": 97, "tokens/total": 3121000, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 44328} +{"epoch": 0.3828125, "grad_norm": 0.6813939213752747, "learning_rate": 9.523275153154695e-05, "loss": 0.03198523819446564, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.0325, "step": 98, "tokens/total": 3152272, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 44778} +{"epoch": 0.38671875, "grad_norm": 0.7792304158210754, "learning_rate": 9.51018809682839e-05, "loss": 0.032100509852170944, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.03262, "step": 99, "tokens/total": 3185480, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 45225} +{"epoch": 0.390625, "grad_norm": 1.1086724996566772, "learning_rate": 9.49693416020645e-05, "loss": 0.017490848898887634, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.01764, "step": 100, "tokens/total": 3219360, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 45678} +{"epoch": 0.39453125, "grad_norm": 0.43673601746559143, "learning_rate": 9.483513894839276e-05, "loss": 0.008856178261339664, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.0089, "step": 101, "tokens/total": 3251368, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 46125} +{"epoch": 0.3984375, "grad_norm": 0.3796325922012329, "learning_rate": 9.469927859198888e-05, "loss": 0.01828189752995968, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01845, "step": 102, "tokens/total": 3284184, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 46612} +{"epoch": 0.40234375, "grad_norm": 1.1521340608596802, "learning_rate": 9.456176618655689e-05, "loss": 0.013203443959355354, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.01329, "step": 103, "tokens/total": 3314752, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 47059} +{"epoch": 0.40625, "grad_norm": 2.012972593307495, "learning_rate": 9.442260745454927e-05, "loss": 0.018075652420520782, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.01824, "step": 104, "tokens/total": 3349736, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 47536} +{"epoch": 0.41015625, "grad_norm": 0.6120038628578186, "learning_rate": 9.428180818692884e-05, "loss": 0.013070151209831238, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.01316, "step": 105, "tokens/total": 3384224, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 48037} +{"epoch": 0.4140625, "grad_norm": 2.16605281829834, "learning_rate": 9.413937424292791e-05, "loss": 0.015707947313785553, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.01583, "step": 106, "tokens/total": 3420808, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 48491} +{"epoch": 0.41796875, "grad_norm": 0.759328305721283, "learning_rate": 9.399531154980424e-05, "loss": 0.017839325591921806, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.018, "step": 107, "tokens/total": 3453512, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 48940} +{"epoch": 0.421875, "grad_norm": 0.6940140724182129, "learning_rate": 9.384962610259455e-05, "loss": 0.023916447535157204, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.0242, "step": 108, "tokens/total": 3486192, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 49389} +{"epoch": 0.42578125, "grad_norm": 0.5327094793319702, "learning_rate": 9.370232396386494e-05, "loss": 0.013307668268680573, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.0134, "step": 109, "tokens/total": 3515384, "tokens/train_per_sec_per_gpu": 10.34, "tokens/trainable": 49838} +{"epoch": 0.4296875, "grad_norm": 1.2897160053253174, "learning_rate": 9.355341126345868e-05, "loss": 0.017657041549682617, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01781, "step": 110, "tokens/total": 3548264, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 50310} +{"epoch": 0.43359375, "grad_norm": 0.9296520948410034, "learning_rate": 9.340289419824107e-05, "loss": 0.02476799301803112, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.02508, "step": 111, "tokens/total": 3579288, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 50755} +{"epoch": 0.4375, "grad_norm": 0.5566288232803345, "learning_rate": 9.325077903184159e-05, "loss": 0.007640478201210499, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00767, "step": 112, "tokens/total": 3611240, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 51213} +{"epoch": 0.44140625, "grad_norm": 0.41159871220588684, "learning_rate": 9.30970720943932e-05, "loss": 0.009857221506536007, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00991, "step": 113, "tokens/total": 3644136, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 51660} +{"epoch": 0.4453125, "grad_norm": 0.6783087849617004, "learning_rate": 9.2941779782269e-05, "loss": 0.011181277222931385, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01124, "step": 114, "tokens/total": 3676368, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 52133} +{"epoch": 0.44921875, "grad_norm": 0.48748087882995605, "learning_rate": 9.278490855781596e-05, "loss": 0.010479525662958622, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.01053, "step": 115, "tokens/total": 3707896, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 52580} +{"epoch": 0.453125, "grad_norm": 0.6105250716209412, "learning_rate": 9.262646494908604e-05, "loss": 0.01640789397060871, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.01654, "step": 116, "tokens/total": 3742296, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 53033} +{"epoch": 0.45703125, "grad_norm": 0.9492978453636169, "learning_rate": 9.246645554956457e-05, "loss": 0.014018632471561432, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01412, "step": 117, "tokens/total": 3776456, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 53517} +{"epoch": 0.4609375, "grad_norm": 1.3093700408935547, "learning_rate": 9.230488701789578e-05, "loss": 0.011019747704267502, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.01108, "step": 118, "tokens/total": 3811688, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 53967} +{"epoch": 0.46484375, "grad_norm": 0.20484071969985962, "learning_rate": 9.214176607760577e-05, "loss": 0.004595973528921604, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00461, "step": 119, "tokens/total": 3847200, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 54430} +{"epoch": 0.46875, "grad_norm": 0.46087029576301575, "learning_rate": 9.197709951682268e-05, "loss": 0.012343390844762325, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.01242, "step": 120, "tokens/total": 3877800, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 54896} +{"epoch": 0.47265625, "grad_norm": 0.6300392150878906, "learning_rate": 9.181089418799428e-05, "loss": 0.027114303782582283, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.02749, "step": 121, "tokens/total": 3910976, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 55379} +{"epoch": 0.4765625, "grad_norm": 0.31465470790863037, "learning_rate": 9.164315700760271e-05, "loss": 0.012830860912799835, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01291, "step": 122, "tokens/total": 3943768, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 55803} +{"epoch": 0.48046875, "grad_norm": 0.21842315793037415, "learning_rate": 9.147389495587671e-05, "loss": 0.009618932381272316, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00967, "step": 123, "tokens/total": 3973816, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 56249} +{"epoch": 0.484375, "grad_norm": 0.225703626871109, "learning_rate": 9.130311507650116e-05, "loss": 0.009289463981986046, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00933, "step": 124, "tokens/total": 4004704, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 56713} +{"epoch": 0.48828125, "grad_norm": 0.32876870036125183, "learning_rate": 9.113082447632394e-05, "loss": 0.00945611298084259, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.0095, "step": 125, "tokens/total": 4036040, "tokens/train_per_sec_per_gpu": 10.53, "tokens/trainable": 57196} +{"epoch": 0.4921875, "grad_norm": 1.0132309198379517, "learning_rate": 9.09570303250602e-05, "loss": 0.024239955469965935, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.02454, "step": 126, "tokens/total": 4066792, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 57680} +{"epoch": 0.49609375, "grad_norm": 0.1759786158800125, "learning_rate": 9.078173985499394e-05, "loss": 0.00530450651422143, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00532, "step": 127, "tokens/total": 4096112, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 58110} +{"epoch": 0.5, "grad_norm": 0.22688116133213043, "learning_rate": 9.060496036067713e-05, "loss": 0.005607918836176395, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00562, "step": 128, "tokens/total": 4131472, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 58534} +{"epoch": 0.50390625, "grad_norm": 0.28071993589401245, "learning_rate": 9.042669919862615e-05, "loss": 0.0075322771444916725, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00756, "step": 129, "tokens/total": 4162912, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 58998} +{"epoch": 0.5078125, "grad_norm": 0.5171173214912415, "learning_rate": 9.024696378701557e-05, "loss": 0.027061067521572113, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.02743, "step": 130, "tokens/total": 4195768, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 59448} +{"epoch": 0.51171875, "grad_norm": 0.11980751901865005, "learning_rate": 9.006576160536948e-05, "loss": 0.002315214602276683, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00232, "step": 131, "tokens/total": 4227192, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 59906} +{"epoch": 0.515625, "grad_norm": 0.3140820860862732, "learning_rate": 8.988310019425035e-05, "loss": 0.009271485731005669, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00931, "step": 132, "tokens/total": 4263304, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 60350} +{"epoch": 0.51953125, "grad_norm": 0.23610679805278778, "learning_rate": 8.969898715494506e-05, "loss": 0.005162850953638554, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00518, "step": 133, "tokens/total": 4297352, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 60831} +{"epoch": 0.5234375, "grad_norm": 0.36976727843284607, "learning_rate": 8.951343014914869e-05, "loss": 0.01748572662472725, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.01764, "step": 134, "tokens/total": 4332152, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 61305} +{"epoch": 0.52734375, "grad_norm": 0.5219255089759827, "learning_rate": 8.932643689864568e-05, "loss": 0.013282082974910736, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.01337, "step": 135, "tokens/total": 4365272, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 61792} +{"epoch": 0.53125, "grad_norm": 0.13121388852596283, "learning_rate": 8.913801518498845e-05, "loss": 0.001823660684749484, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00183, "step": 136, "tokens/total": 4399928, "tokens/train_per_sec_per_gpu": 6.93, "tokens/trainable": 62253} +{"epoch": 0.53515625, "grad_norm": 0.4312112033367157, "learning_rate": 8.894817284917364e-05, "loss": 0.009758121334016323, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00981, "step": 137, "tokens/total": 4434336, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 62707} +{"epoch": 0.5390625, "grad_norm": 0.18602800369262695, "learning_rate": 8.875691779131569e-05, "loss": 0.0026344344951212406, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00264, "step": 138, "tokens/total": 4468872, "tokens/train_per_sec_per_gpu": 6.43, "tokens/trainable": 63145} +{"epoch": 0.54296875, "grad_norm": 2.2483201026916504, "learning_rate": 8.856425797031829e-05, "loss": 0.005801444407552481, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00582, "step": 139, "tokens/total": 4502416, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 63587} +{"epoch": 0.546875, "grad_norm": 0.22327224910259247, "learning_rate": 8.837020140354295e-05, "loss": 0.004072893410921097, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00408, "step": 140, "tokens/total": 4536024, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 64052} +{"epoch": 0.55078125, "grad_norm": 0.1498948186635971, "learning_rate": 8.817475616647554e-05, "loss": 0.0018481060396879911, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00185, "step": 141, "tokens/total": 4568960, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 64526} +{"epoch": 0.5546875, "grad_norm": 0.1717899888753891, "learning_rate": 8.797793039239017e-05, "loss": 0.0028472652193158865, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00285, "step": 142, "tokens/total": 4601656, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 64983} +{"epoch": 0.55859375, "grad_norm": 0.6536439061164856, "learning_rate": 8.777973227201069e-05, "loss": 0.007507209200412035, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00754, "step": 143, "tokens/total": 4636136, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 65492} +{"epoch": 0.5625, "grad_norm": 0.16473738849163055, "learning_rate": 8.758017005316988e-05, "loss": 0.003382556140422821, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00339, "step": 144, "tokens/total": 4670160, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 65925} +{"epoch": 0.56640625, "grad_norm": 0.28885847330093384, "learning_rate": 8.737925204046629e-05, "loss": 0.0034196199849247932, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.00343, "step": 145, "tokens/total": 4703792, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 66383} +{"epoch": 0.5703125, "grad_norm": 0.019681675359606743, "learning_rate": 8.717698659491851e-05, "loss": 0.0004782151081599295, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00048, "step": 146, "tokens/total": 4737640, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 66840} +{"epoch": 0.57421875, "grad_norm": 1.1160835027694702, "learning_rate": 8.697338213361735e-05, "loss": 0.02048776112496853, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.0207, "step": 147, "tokens/total": 4770704, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 67297} +{"epoch": 0.578125, "grad_norm": 0.22596076130867004, "learning_rate": 8.676844712937552e-05, "loss": 0.005250786896795034, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00526, "step": 148, "tokens/total": 4807376, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 67769} +{"epoch": 0.58203125, "grad_norm": 0.3786258101463318, "learning_rate": 8.656219011037509e-05, "loss": 0.008804867044091225, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00884, "step": 149, "tokens/total": 4840728, "tokens/train_per_sec_per_gpu": 6.63, "tokens/trainable": 68257} +{"epoch": 0.5859375, "grad_norm": 0.3972109258174896, "learning_rate": 8.63546196598125e-05, "loss": 0.013099108822643757, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.01319, "step": 150, "tokens/total": 4873880, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 68706} +{"epoch": 0.58984375, "grad_norm": 0.4304192066192627, "learning_rate": 8.614574441554145e-05, "loss": 0.01662297360599041, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.01676, "step": 151, "tokens/total": 4904512, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 69131} +{"epoch": 0.59375, "grad_norm": 0.546416699886322, "learning_rate": 8.593557306971349e-05, "loss": 0.011733943596482277, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.0118, "step": 152, "tokens/total": 4937048, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 69586} +{"epoch": 0.59765625, "grad_norm": 0.1932176798582077, "learning_rate": 8.572411436841618e-05, "loss": 0.0046434164978563786, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00465, "step": 153, "tokens/total": 4968224, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 70061} +{"epoch": 0.6015625, "grad_norm": 0.16530324518680573, "learning_rate": 8.551137711130922e-05, "loss": 0.004032410681247711, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00404, "step": 154, "tokens/total": 4997272, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 70467} +{"epoch": 0.60546875, "grad_norm": 0.13440652191638947, "learning_rate": 8.529737015125824e-05, "loss": 0.003304606769233942, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00331, "step": 155, "tokens/total": 5031072, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 70933} +{"epoch": 0.609375, "grad_norm": 0.362918496131897, "learning_rate": 8.508210239396639e-05, "loss": 0.009914370253682137, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00996, "step": 156, "tokens/total": 5063808, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 71382} +{"epoch": 0.61328125, "grad_norm": 0.22061100602149963, "learning_rate": 8.486558279760375e-05, "loss": 0.006216924171894789, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00624, "step": 157, "tokens/total": 5094032, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 71808} +{"epoch": 0.6171875, "grad_norm": 0.876031219959259, "learning_rate": 8.464782037243449e-05, "loss": 0.030833903700113297, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.03131, "step": 158, "tokens/total": 5125664, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 72249} +{"epoch": 0.62109375, "grad_norm": 0.19932889938354492, "learning_rate": 8.442882418044202e-05, "loss": 0.004941292107105255, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00495, "step": 159, "tokens/total": 5156952, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 72726} +{"epoch": 0.625, "grad_norm": 0.6232088804244995, "learning_rate": 8.420860333495179e-05, "loss": 0.007052659057080746, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00708, "step": 160, "tokens/total": 5187120, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 73148} +{"epoch": 0.62890625, "grad_norm": 0.02161034196615219, "learning_rate": 8.398716700025208e-05, "loss": 0.0005191901582293212, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00052, "step": 161, "tokens/total": 5219080, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 73600} +{"epoch": 0.6328125, "grad_norm": 0.10537007451057434, "learning_rate": 8.376452439121266e-05, "loss": 0.0018163733184337616, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.00182, "step": 162, "tokens/total": 5248680, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 74068} +{"epoch": 0.63671875, "grad_norm": 0.37560996413230896, "learning_rate": 8.354068477290124e-05, "loss": 0.004570978693664074, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00458, "step": 163, "tokens/total": 5280104, "tokens/train_per_sec_per_gpu": 10.18, "tokens/trainable": 74527} +{"epoch": 0.640625, "grad_norm": 0.3161824941635132, "learning_rate": 8.331565746019807e-05, "loss": 0.00664493115618825, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00667, "step": 164, "tokens/total": 5311328, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 74992} +{"epoch": 0.64453125, "grad_norm": 0.6636349558830261, "learning_rate": 8.308945181740812e-05, "loss": 0.011098379269242287, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.01116, "step": 165, "tokens/total": 5343960, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 75442} +{"epoch": 0.6484375, "grad_norm": 0.10276741534471512, "learning_rate": 8.286207725787153e-05, "loss": 0.0017200750298798084, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00172, "step": 166, "tokens/total": 5377312, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 75887} +{"epoch": 0.65234375, "grad_norm": 0.9044793844223022, "learning_rate": 8.263354324357182e-05, "loss": 0.018061885610222816, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.01823, "step": 167, "tokens/total": 5409824, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 76331} +{"epoch": 0.65625, "grad_norm": 0.2681965231895447, "learning_rate": 8.240385928474219e-05, "loss": 0.0037490096874535084, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00376, "step": 168, "tokens/total": 5440904, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 76745} +{"epoch": 0.66015625, "grad_norm": 0.08091765642166138, "learning_rate": 8.217303493946967e-05, "loss": 0.0013793734833598137, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00138, "step": 169, "tokens/total": 5474920, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 77201} +{"epoch": 0.6640625, "grad_norm": 0.4137412905693054, "learning_rate": 8.194107981329746e-05, "loss": 0.010631698183715343, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01069, "step": 170, "tokens/total": 5511856, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 77655} +{"epoch": 0.66796875, "grad_norm": 0.05574122816324234, "learning_rate": 8.170800355882518e-05, "loss": 0.001237829215824604, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00124, "step": 171, "tokens/total": 5544104, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 78122} +{"epoch": 0.671875, "grad_norm": 1.3937852382659912, "learning_rate": 8.147381587530713e-05, "loss": 0.006453222129493952, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00647, "step": 172, "tokens/total": 5577344, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 78576} +{"epoch": 0.67578125, "grad_norm": 0.4697269797325134, "learning_rate": 8.123852650824877e-05, "loss": 0.011561068706214428, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.01163, "step": 173, "tokens/total": 5610024, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 79059} +{"epoch": 0.6796875, "grad_norm": 0.27886340022087097, "learning_rate": 8.100214524900103e-05, "loss": 0.008517350070178509, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00855, "step": 174, "tokens/total": 5643184, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 79498} +{"epoch": 0.68359375, "grad_norm": 0.31607067584991455, "learning_rate": 8.076468193435301e-05, "loss": 0.00838526338338852, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00842, "step": 175, "tokens/total": 5672824, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 79922} +{"epoch": 0.6875, "grad_norm": 0.2793390452861786, "learning_rate": 8.052614644612253e-05, "loss": 0.007038774900138378, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00706, "step": 176, "tokens/total": 5705576, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 80383} +{"epoch": 0.69140625, "grad_norm": 0.13339534401893616, "learning_rate": 8.028654871074489e-05, "loss": 0.0025923841167241335, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0026, "step": 177, "tokens/total": 5737840, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 80824} +{"epoch": 0.6953125, "grad_norm": 0.11088567227125168, "learning_rate": 8.004589869885986e-05, "loss": 0.002684911247342825, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00269, "step": 178, "tokens/total": 5767552, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 81243} +{"epoch": 0.69921875, "grad_norm": 0.19802607595920563, "learning_rate": 7.980420642489674e-05, "loss": 0.004618468694388866, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00463, "step": 179, "tokens/total": 5802512, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 81716} +{"epoch": 0.703125, "grad_norm": 0.1512758582830429, "learning_rate": 7.95614819466576e-05, "loss": 0.003976650536060333, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00398, "step": 180, "tokens/total": 5834984, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 82181} +{"epoch": 0.70703125, "grad_norm": 0.12549331784248352, "learning_rate": 7.931773536489872e-05, "loss": 0.002624727785587311, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00263, "step": 181, "tokens/total": 5867336, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 82626} +{"epoch": 0.7109375, "grad_norm": 0.40909016132354736, "learning_rate": 7.907297682291035e-05, "loss": 0.004215116612613201, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00422, "step": 182, "tokens/total": 5899232, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 83089} +{"epoch": 0.71484375, "grad_norm": 0.249625563621521, "learning_rate": 7.882721650609442e-05, "loss": 0.0038673002272844315, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00387, "step": 183, "tokens/total": 5934816, "tokens/train_per_sec_per_gpu": 6.73, "tokens/trainable": 83590} +{"epoch": 0.71875, "grad_norm": 0.7493933439254761, "learning_rate": 7.85804646415409e-05, "loss": 0.021326754242181778, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.02156, "step": 184, "tokens/total": 5968120, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 84046} +{"epoch": 0.72265625, "grad_norm": 0.24347946047782898, "learning_rate": 7.833273149760207e-05, "loss": 0.003108662785962224, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00311, "step": 185, "tokens/total": 5998600, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 84517} +{"epoch": 0.7265625, "grad_norm": 0.4345245957374573, "learning_rate": 7.808402738346527e-05, "loss": 0.007036128547042608, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00706, "step": 186, "tokens/total": 6033832, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 84974} +{"epoch": 0.73046875, "grad_norm": 0.009984391741454601, "learning_rate": 7.783436264872382e-05, "loss": 0.00011797271145042032, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00012, "step": 187, "tokens/total": 6068192, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 85460} +{"epoch": 0.734375, "grad_norm": 0.7561762928962708, "learning_rate": 7.758374768294647e-05, "loss": 0.026290412992239, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.02664, "step": 188, "tokens/total": 6101464, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 85939} +{"epoch": 0.73828125, "grad_norm": 0.16268691420555115, "learning_rate": 7.733219291524489e-05, "loss": 0.003222685307264328, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00323, "step": 189, "tokens/total": 6133008, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 86377} +{"epoch": 0.7421875, "grad_norm": 0.02063925191760063, "learning_rate": 7.707970881383977e-05, "loss": 0.00042931700590997934, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.00043, "step": 190, "tokens/total": 6163048, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 86834} +{"epoch": 0.74609375, "grad_norm": 0.4501875042915344, "learning_rate": 7.682630588562518e-05, "loss": 0.010208030231297016, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01026, "step": 191, "tokens/total": 6196072, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 87265} +{"epoch": 0.75, "grad_norm": 0.24536798894405365, "learning_rate": 7.657199467573129e-05, "loss": 0.0063636526465415955, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00638, "step": 192, "tokens/total": 6227312, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 87747} +{"epoch": 0.75390625, "grad_norm": 0.08779238164424896, "learning_rate": 7.631678576708561e-05, "loss": 0.0019332020310685039, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00194, "step": 193, "tokens/total": 6260240, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 88239} +{"epoch": 0.7578125, "grad_norm": 0.1274586021900177, "learning_rate": 7.606068977997255e-05, "loss": 0.0017910473980009556, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00179, "step": 194, "tokens/total": 6294280, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 88718} +{"epoch": 0.76171875, "grad_norm": 0.41667985916137695, "learning_rate": 7.580371737159148e-05, "loss": 0.013775545172393322, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.01387, "step": 195, "tokens/total": 6323368, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 89129} +{"epoch": 0.765625, "grad_norm": 0.15013830363750458, "learning_rate": 7.554587923561324e-05, "loss": 0.0019012435805052519, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.0019, "step": 196, "tokens/total": 6354960, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 89567} +{"epoch": 0.76953125, "grad_norm": 0.5238422751426697, "learning_rate": 7.528718610173511e-05, "loss": 0.015604042448103428, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.01573, "step": 197, "tokens/total": 6387208, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 89988} +{"epoch": 0.7734375, "grad_norm": 0.11190782487392426, "learning_rate": 7.502764873523431e-05, "loss": 0.0022242709528654814, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00223, "step": 198, "tokens/total": 6418720, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 90434} +{"epoch": 0.77734375, "grad_norm": 0.23360835015773773, "learning_rate": 7.476727793652011e-05, "loss": 0.00534425862133503, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00536, "step": 199, "tokens/total": 6449184, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 90913} +{"epoch": 0.78125, "grad_norm": 0.11427053064107895, "learning_rate": 7.450608454068415e-05, "loss": 0.002508576726540923, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00251, "step": 200, "tokens/total": 6481824, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 91355} +{"epoch": 0.78515625, "grad_norm": 0.20730647444725037, "learning_rate": 7.424407941704987e-05, "loss": 0.003914813976734877, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00392, "step": 201, "tokens/total": 6515896, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 91856} +{"epoch": 0.7890625, "grad_norm": 0.5530998706817627, "learning_rate": 7.398127346871986e-05, "loss": 0.011919579468667507, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.01199, "step": 202, "tokens/total": 6548784, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 92311} +{"epoch": 0.79296875, "grad_norm": 0.028012696653604507, "learning_rate": 7.371767763212238e-05, "loss": 0.0006968079833313823, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.0007, "step": 203, "tokens/total": 6580256, "tokens/train_per_sec_per_gpu": 10.08, "tokens/trainable": 92764} +{"epoch": 0.796875, "grad_norm": 0.09572044759988785, "learning_rate": 7.345330287655617e-05, "loss": 0.0017665064660832286, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00177, "step": 204, "tokens/total": 6613976, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 93227} +{"epoch": 0.80078125, "grad_norm": 0.33907002210617065, "learning_rate": 7.31881602037339e-05, "loss": 0.008199089206755161, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.00823, "step": 205, "tokens/total": 6644552, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 93675} +{"epoch": 0.8046875, "grad_norm": 0.34270069003105164, "learning_rate": 7.29222606473245e-05, "loss": 0.00607380922883749, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00609, "step": 206, "tokens/total": 6675056, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 94120} +{"epoch": 0.80859375, "grad_norm": 0.19321097433567047, "learning_rate": 7.265561527249383e-05, "loss": 0.0018398615065962076, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00184, "step": 207, "tokens/total": 6706624, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 94557} +{"epoch": 0.8125, "grad_norm": 0.24629154801368713, "learning_rate": 7.238823517544436e-05, "loss": 0.002977657364681363, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00298, "step": 208, "tokens/total": 6741008, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 95035} +{"epoch": 0.81640625, "grad_norm": 0.05591266602277756, "learning_rate": 7.212013148295333e-05, "loss": 0.0005803058156743646, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00058, "step": 209, "tokens/total": 6774872, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 95517} +{"epoch": 0.8203125, "grad_norm": 0.40049299597740173, "learning_rate": 7.185131535190975e-05, "loss": 0.010907408781349659, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.01097, "step": 210, "tokens/total": 6805816, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 96026} +{"epoch": 0.82421875, "grad_norm": 0.04291468486189842, "learning_rate": 7.158179796885005e-05, "loss": 0.00023652704840060323, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00024, "step": 211, "tokens/total": 6837592, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 96477} +{"epoch": 0.828125, "grad_norm": 0.042685918509960175, "learning_rate": 7.131159054949273e-05, "loss": 0.00041452201548963785, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00041, "step": 212, "tokens/total": 6874384, "tokens/train_per_sec_per_gpu": 7.12, "tokens/trainable": 96951} +{"epoch": 0.83203125, "grad_norm": 0.1036536693572998, "learning_rate": 7.104070433827139e-05, "loss": 0.00066697335569188, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00067, "step": 213, "tokens/total": 6905872, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 97350} +{"epoch": 0.8359375, "grad_norm": 0.0169181227684021, "learning_rate": 7.076915060786705e-05, "loss": 0.0001940245711011812, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.00019, "step": 214, "tokens/total": 6937984, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 97792} +{"epoch": 0.83984375, "grad_norm": 0.82615727186203, "learning_rate": 7.049694065873882e-05, "loss": 0.021484920755028725, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.02172, "step": 215, "tokens/total": 6970704, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 98257} +{"epoch": 0.84375, "grad_norm": 1.3846848011016846, "learning_rate": 7.022408581865382e-05, "loss": 0.019321203231811523, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.01951, "step": 216, "tokens/total": 7004616, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 98731} +{"epoch": 0.84765625, "grad_norm": 0.13795657455921173, "learning_rate": 6.99505974422157e-05, "loss": 0.0009222462540492415, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00092, "step": 217, "tokens/total": 7036880, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 99212} +{"epoch": 0.8515625, "grad_norm": 0.36262020468711853, "learning_rate": 6.967648691039213e-05, "loss": 0.006511043291538954, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00653, "step": 218, "tokens/total": 7068208, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 99654} +{"epoch": 0.85546875, "grad_norm": 0.04165354743599892, "learning_rate": 6.940176563004123e-05, "loss": 0.00048615041305311024, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00049, "step": 219, "tokens/total": 7098112, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 100086} +{"epoch": 0.859375, "grad_norm": 0.2763936221599579, "learning_rate": 6.912644503343682e-05, "loss": 0.005121700465679169, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00513, "step": 220, "tokens/total": 7132432, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 100524} +{"epoch": 0.86328125, "grad_norm": 0.05739597603678703, "learning_rate": 6.885053657779273e-05, "loss": 0.0007834231364540756, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.00078, "step": 221, "tokens/total": 7164016, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 100996} +{"epoch": 0.8671875, "grad_norm": 0.3846910297870636, "learning_rate": 6.857405174478604e-05, "loss": 0.0046044462360441685, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00462, "step": 222, "tokens/total": 7198344, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 101449} +{"epoch": 0.87109375, "grad_norm": 0.06117518991231918, "learning_rate": 6.82970020400792e-05, "loss": 0.0009637732291594148, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00096, "step": 223, "tokens/total": 7232192, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 101905} +{"epoch": 0.875, "grad_norm": 0.434199720621109, "learning_rate": 6.801939899284132e-05, "loss": 0.014820214360952377, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.01493, "step": 224, "tokens/total": 7266984, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 102411} +{"epoch": 0.87890625, "grad_norm": 0.07270750403404236, "learning_rate": 6.774125415526827e-05, "loss": 0.0010560404043644667, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00106, "step": 225, "tokens/total": 7299512, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 102882} +{"epoch": 0.8828125, "grad_norm": 0.04514242708683014, "learning_rate": 6.746257910210214e-05, "loss": 0.0007607759325765073, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00076, "step": 226, "tokens/total": 7331120, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 103332} +{"epoch": 0.88671875, "grad_norm": 0.4789307713508606, "learning_rate": 6.718338543014937e-05, "loss": 0.006780568044632673, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.0068, "step": 227, "tokens/total": 7360504, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 103790} +{"epoch": 0.890625, "grad_norm": 0.11853453516960144, "learning_rate": 6.69036847577983e-05, "loss": 0.0020374145824462175, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00204, "step": 228, "tokens/total": 7390760, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 104246} +{"epoch": 0.89453125, "grad_norm": 0.10299761593341827, "learning_rate": 6.662348872453553e-05, "loss": 0.00325782154686749, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00326, "step": 229, "tokens/total": 7423352, "tokens/train_per_sec_per_gpu": 10.49, "tokens/trainable": 104707} +{"epoch": 0.8984375, "grad_norm": 0.05408736690878868, "learning_rate": 6.63428089904618e-05, "loss": 0.0005067445454187691, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00051, "step": 230, "tokens/total": 7457152, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 105167} +{"epoch": 0.90234375, "grad_norm": 0.20916469395160675, "learning_rate": 6.60616572358065e-05, "loss": 0.003866552608087659, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00387, "step": 231, "tokens/total": 7488648, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 105576} +{"epoch": 0.90625, "grad_norm": 0.06664387136697769, "learning_rate": 6.578004516044172e-05, "loss": 0.0005651679821312428, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00057, "step": 232, "tokens/total": 7522160, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 106043} +{"epoch": 0.91015625, "grad_norm": 0.23493139445781708, "learning_rate": 6.549798448339548e-05, "loss": 0.0038330599199980497, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00384, "step": 233, "tokens/total": 7557336, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 106506} +{"epoch": 0.9140625, "grad_norm": 0.10086486488580704, "learning_rate": 6.521548694236384e-05, "loss": 0.0009348825551569462, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00094, "step": 234, "tokens/total": 7590432, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 106951} +{"epoch": 0.91796875, "grad_norm": 0.07934077084064484, "learning_rate": 6.493256429322259e-05, "loss": 0.0012429774506017566, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00124, "step": 235, "tokens/total": 7623240, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 107382} +{"epoch": 0.921875, "grad_norm": 0.5034355521202087, "learning_rate": 6.464922830953799e-05, "loss": 0.0067983511835336685, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00682, "step": 236, "tokens/total": 7657968, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 107814} +{"epoch": 0.92578125, "grad_norm": 0.37809640169143677, "learning_rate": 6.436549078207688e-05, "loss": 0.011838007718324661, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.01191, "step": 237, "tokens/total": 7689320, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 108274} +{"epoch": 0.9296875, "grad_norm": 0.1345602422952652, "learning_rate": 6.408136351831592e-05, "loss": 0.002168966457247734, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00217, "step": 238, "tokens/total": 7721568, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 108714} +{"epoch": 0.93359375, "grad_norm": 0.026447711512446404, "learning_rate": 6.379685834195036e-05, "loss": 0.0002502241695765406, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00025, "step": 239, "tokens/total": 7756464, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 109220} +{"epoch": 0.9375, "grad_norm": 0.014654562808573246, "learning_rate": 6.351198709240186e-05, "loss": 0.0001759407459758222, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00018, "step": 240, "tokens/total": 7791616, "tokens/train_per_sec_per_gpu": 7.01, "tokens/trainable": 109672} +{"epoch": 0.94140625, "grad_norm": 0.1743643879890442, "learning_rate": 6.32267616243259e-05, "loss": 0.001719662337563932, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00172, "step": 241, "tokens/total": 7823520, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 110135} +{"epoch": 0.9453125, "grad_norm": 0.09445378929376602, "learning_rate": 6.294119380711849e-05, "loss": 0.0006026530172675848, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.0006, "step": 242, "tokens/total": 7854424, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 110563} +{"epoch": 0.94921875, "grad_norm": 0.39402925968170166, "learning_rate": 6.265529552442209e-05, "loss": 0.006912100128829479, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00694, "step": 243, "tokens/total": 7887160, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 111049} +{"epoch": 0.953125, "grad_norm": 0.027886854484677315, "learning_rate": 6.236907867363127e-05, "loss": 0.00028006936190649867, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00028, "step": 244, "tokens/total": 7918736, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 111495} +{"epoch": 0.95703125, "grad_norm": 0.062198735773563385, "learning_rate": 6.208255516539749e-05, "loss": 0.0008703722269274294, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00087, "step": 245, "tokens/total": 7949448, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 111968} +{"epoch": 0.9609375, "grad_norm": 0.019871046766638756, "learning_rate": 6.179573692313344e-05, "loss": 0.0002271030389238149, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00023, "step": 246, "tokens/total": 7984680, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 112416} +{"epoch": 0.96484375, "grad_norm": 0.4272684156894684, "learning_rate": 6.150863588251694e-05, "loss": 0.0031889001838862896, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00319, "step": 247, "tokens/total": 8019816, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 112882} +{"epoch": 0.96875, "grad_norm": 0.0234213937073946, "learning_rate": 6.122126399099419e-05, "loss": 0.00022354506654664874, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00022, "step": 248, "tokens/total": 8056272, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 113390} +{"epoch": 0.97265625, "grad_norm": 0.44489040970802307, "learning_rate": 6.0933633207282615e-05, "loss": 0.015655791386961937, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.01578, "step": 249, "tokens/total": 8090224, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 113904} +{"epoch": 0.9765625, "grad_norm": 0.8477278351783752, "learning_rate": 6.064575550087316e-05, "loss": 0.00771808298304677, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00775, "step": 250, "tokens/total": 8124528, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 114351} +{"epoch": 0.98046875, "grad_norm": 0.9443350434303284, "learning_rate": 6.0357642851532245e-05, "loss": 0.021138010546565056, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.02136, "step": 251, "tokens/total": 8159688, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 114842} +{"epoch": 0.984375, "grad_norm": 0.057521965354681015, "learning_rate": 6.0069307248803294e-05, "loss": 0.0003536642761901021, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00035, "step": 252, "tokens/total": 8191800, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 115263} +{"epoch": 0.98828125, "grad_norm": 0.03412086144089699, "learning_rate": 5.9780760691507635e-05, "loss": 0.0005501940613612533, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00055, "step": 253, "tokens/total": 8221752, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 115703} +{"epoch": 0.9921875, "grad_norm": 0.17940017580986023, "learning_rate": 5.9492015187245334e-05, "loss": 0.0019165546400472522, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00192, "step": 254, "tokens/total": 8255648, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 116157} +{"epoch": 0.99609375, "grad_norm": 0.16422562301158905, "learning_rate": 5.920308275189541e-05, "loss": 0.002640434307977557, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00264, "step": 255, "tokens/total": 8286664, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 116567} +{"epoch": 1.0, "grad_norm": 0.03326995298266411, "learning_rate": 5.8913975409115874e-05, "loss": 0.0007508259732276201, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00075, "step": 256, "tokens/total": 8318368, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 117030} +{"epoch": 1.00390625, "grad_norm": 0.2603195905685425, "learning_rate": 5.8624705189843395e-05, "loss": 0.006895578000694513, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00692, "step": 257, "tokens/total": 8353336, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 117517} +{"epoch": 1.0078125, "grad_norm": 0.25259652733802795, "learning_rate": 5.833528413179249e-05, "loss": 0.005110118072479963, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00512, "step": 258, "tokens/total": 8389112, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 118015} +{"epoch": 1.01171875, "grad_norm": 0.33942756056785583, "learning_rate": 5.80457242789548e-05, "loss": 0.007950395345687866, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00798, "step": 259, "tokens/total": 8426000, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 118520} +{"epoch": 1.015625, "grad_norm": 0.07626030594110489, "learning_rate": 5.77560376810977e-05, "loss": 0.0016670054756104946, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.00167, "step": 260, "tokens/total": 8458568, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 118992} +{"epoch": 1.01953125, "grad_norm": 0.08212272077798843, "learning_rate": 5.7466236393263005e-05, "loss": 0.0008444542763754725, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00084, "step": 261, "tokens/total": 8490840, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 119438} +{"epoch": 1.0234375, "grad_norm": 0.09348843991756439, "learning_rate": 5.717633247526522e-05, "loss": 0.001244512852281332, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00125, "step": 262, "tokens/total": 8521792, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 119881} +{"epoch": 1.02734375, "grad_norm": 0.028138723224401474, "learning_rate": 5.688633799118971e-05, "loss": 0.0002603459870442748, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00026, "step": 263, "tokens/total": 8551336, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 120285} +{"epoch": 1.03125, "grad_norm": 0.11296309530735016, "learning_rate": 5.659626500889066e-05, "loss": 0.001041956478729844, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00104, "step": 264, "tokens/total": 8579872, "tokens/train_per_sec_per_gpu": 7.02, "tokens/trainable": 120755} +{"epoch": 1.03515625, "grad_norm": 0.023632941767573357, "learning_rate": 5.6306125599488905e-05, "loss": 0.0002985709288623184, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.0003, "step": 265, "tokens/total": 8608920, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 121194} +{"epoch": 1.0390625, "grad_norm": 0.003447567345574498, "learning_rate": 5.601593183686955e-05, "loss": 6.361484702210873e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00006, "step": 266, "tokens/total": 8643600, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 121670} +{"epoch": 1.04296875, "grad_norm": 0.24759778380393982, "learning_rate": 5.572569579717961e-05, "loss": 0.003281570039689541, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00329, "step": 267, "tokens/total": 8676032, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 122142} +{"epoch": 1.046875, "grad_norm": 0.04610765352845192, "learning_rate": 5.543542955832538e-05, "loss": 0.0005362502415664494, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00054, "step": 268, "tokens/total": 8713240, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 122585} +{"epoch": 1.05078125, "grad_norm": 0.026085998862981796, "learning_rate": 5.514514519946986e-05, "loss": 0.00037454930134117603, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00037, "step": 269, "tokens/total": 8751664, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 123091} +{"epoch": 1.0546875, "grad_norm": 0.0192096084356308, "learning_rate": 5.485485480053015e-05, "loss": 0.0001594039931660518, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00016, "step": 270, "tokens/total": 8786112, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 123505} +{"epoch": 1.05859375, "grad_norm": 0.003491560695692897, "learning_rate": 5.4564570441674645e-05, "loss": 5.416760905063711e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00005, "step": 271, "tokens/total": 8816920, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 123953} +{"epoch": 1.0625, "grad_norm": 0.07457486540079117, "learning_rate": 5.42743042028204e-05, "loss": 0.0007197184604592621, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00072, "step": 272, "tokens/total": 8851408, "tokens/train_per_sec_per_gpu": 6.81, "tokens/trainable": 124400} +{"epoch": 1.06640625, "grad_norm": 0.04659455642104149, "learning_rate": 5.3984068163130464e-05, "loss": 0.00045268077519722283, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00045, "step": 273, "tokens/total": 8883768, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 124870} +{"epoch": 1.0703125, "grad_norm": 0.2721177637577057, "learning_rate": 5.369387440051111e-05, "loss": 0.002017945284023881, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00202, "step": 274, "tokens/total": 8916200, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 125333} +{"epoch": 1.07421875, "grad_norm": 0.0027952897362411022, "learning_rate": 5.340373499110935e-05, "loss": 4.689697743742727e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00005, "step": 275, "tokens/total": 8949920, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 125834} +{"epoch": 1.078125, "grad_norm": 0.023503581061959267, "learning_rate": 5.3113662008810304e-05, "loss": 0.00015455097309313715, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00015, "step": 276, "tokens/total": 8986448, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 126316} +{"epoch": 1.08203125, "grad_norm": 0.47149085998535156, "learning_rate": 5.282366752473479e-05, "loss": 0.004794555716216564, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00481, "step": 277, "tokens/total": 9020600, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 126798} +{"epoch": 1.0859375, "grad_norm": 0.03939788416028023, "learning_rate": 5.2533763606737005e-05, "loss": 0.000211514241527766, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00021, "step": 278, "tokens/total": 9056040, "tokens/train_per_sec_per_gpu": 6.87, "tokens/trainable": 127223} +{"epoch": 1.08984375, "grad_norm": 0.5559512972831726, "learning_rate": 5.224396231890232e-05, "loss": 0.009909281507134438, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00996, "step": 279, "tokens/total": 9087072, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 127672} +{"epoch": 1.09375, "grad_norm": 0.3564532995223999, "learning_rate": 5.195427572104522e-05, "loss": 0.004840170498937368, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00485, "step": 280, "tokens/total": 9118376, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 128116} +{"epoch": 1.09765625, "grad_norm": 0.2815982699394226, "learning_rate": 5.166471586820751e-05, "loss": 0.0033167318906635046, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00332, "step": 281, "tokens/total": 9152704, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 128589} +{"epoch": 1.1015625, "grad_norm": 0.23720866441726685, "learning_rate": 5.1375294810156615e-05, "loss": 0.0047612437047064304, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00477, "step": 282, "tokens/total": 9184448, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 129036} +{"epoch": 1.10546875, "grad_norm": 0.11451013386249542, "learning_rate": 5.1086024590884144e-05, "loss": 0.0006720614619553089, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00067, "step": 283, "tokens/total": 9217392, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 129485} +{"epoch": 1.109375, "grad_norm": 0.050869300961494446, "learning_rate": 5.079691724810461e-05, "loss": 0.0006329301977530122, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00063, "step": 284, "tokens/total": 9252016, "tokens/train_per_sec_per_gpu": 7.09, "tokens/trainable": 129920} +{"epoch": 1.11328125, "grad_norm": 0.11242621392011642, "learning_rate": 5.0507984812754684e-05, "loss": 0.0007372417603619397, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00074, "step": 285, "tokens/total": 9284008, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 130417} +{"epoch": 1.1171875, "grad_norm": 0.40981343388557434, "learning_rate": 5.021923930849237e-05, "loss": 0.00693802023306489, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00696, "step": 286, "tokens/total": 9315984, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 130903} +{"epoch": 1.12109375, "grad_norm": 0.007026659324765205, "learning_rate": 4.99306927511967e-05, "loss": 7.363961776718497e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00007, "step": 287, "tokens/total": 9347376, "tokens/train_per_sec_per_gpu": 10.4, "tokens/trainable": 131343} +{"epoch": 1.125, "grad_norm": 0.08191800117492676, "learning_rate": 4.964235714846775e-05, "loss": 0.0007918696501292288, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00079, "step": 288, "tokens/total": 9377424, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 131763} +{"epoch": 1.12890625, "grad_norm": 0.010348781943321228, "learning_rate": 4.9354244499126866e-05, "loss": 0.00012956438877154142, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00013, "step": 289, "tokens/total": 9408104, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 132229} +{"epoch": 1.1328125, "grad_norm": 0.03148503229022026, "learning_rate": 4.90663667927174e-05, "loss": 0.00017942361591849476, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00018, "step": 290, "tokens/total": 9439944, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 132678} +{"epoch": 1.13671875, "grad_norm": 0.01813829317688942, "learning_rate": 4.877873600900581e-05, "loss": 0.00015467475168406963, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00015, "step": 291, "tokens/total": 9472104, "tokens/train_per_sec_per_gpu": 6.21, "tokens/trainable": 133136} +{"epoch": 1.140625, "grad_norm": 0.014840344898402691, "learning_rate": 4.849136411748306e-05, "loss": 0.00023142040299717337, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00023, "step": 292, "tokens/total": 9508336, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 133608} +{"epoch": 1.14453125, "grad_norm": 0.35432472825050354, "learning_rate": 4.8204263076866574e-05, "loss": 0.0047523751854896545, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00476, "step": 293, "tokens/total": 9541552, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 134108} +{"epoch": 1.1484375, "grad_norm": 0.012410170398652554, "learning_rate": 4.791744483460251e-05, "loss": 6.488186772912741e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00006, "step": 294, "tokens/total": 9575072, "tokens/train_per_sec_per_gpu": 6.56, "tokens/trainable": 134541} +{"epoch": 1.15234375, "grad_norm": 0.0016024296637624502, "learning_rate": 4.7630921326368736e-05, "loss": 2.826004129019566e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00003, "step": 295, "tokens/total": 9610344, "tokens/train_per_sec_per_gpu": 6.4, "tokens/trainable": 134966} +{"epoch": 1.15625, "grad_norm": 0.012622863985598087, "learning_rate": 4.7344704475577916e-05, "loss": 0.00012520373275037855, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00013, "step": 296, "tokens/total": 9639728, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 135402} +{"epoch": 1.16015625, "grad_norm": 0.11868783831596375, "learning_rate": 4.705880619288153e-05, "loss": 0.0009943883633241057, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00099, "step": 297, "tokens/total": 9671424, "tokens/train_per_sec_per_gpu": 7.08, "tokens/trainable": 135804} +{"epoch": 1.1640625, "grad_norm": 0.0929541289806366, "learning_rate": 4.677323837567412e-05, "loss": 0.00016830363892950118, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00017, "step": 298, "tokens/total": 9700808, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 136231} +{"epoch": 1.16796875, "grad_norm": 0.19813711941242218, "learning_rate": 4.6488012907598146e-05, "loss": 0.0011738959001377225, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00117, "step": 299, "tokens/total": 9732312, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 136705} +{"epoch": 1.171875, "grad_norm": 0.07302466034889221, "learning_rate": 4.620314165804964e-05, "loss": 0.0008872398757375777, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00089, "step": 300, "tokens/total": 9764624, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 137178} +{"epoch": 1.17578125, "grad_norm": 0.024565791711211205, "learning_rate": 4.591863648168407e-05, "loss": 0.00026973665808327496, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00027, "step": 301, "tokens/total": 9798368, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 137643} +{"epoch": 1.1796875, "grad_norm": 0.20254001021385193, "learning_rate": 4.5634509217923135e-05, "loss": 0.0032841493375599384, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00329, "step": 302, "tokens/total": 9830192, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 138078} +{"epoch": 1.18359375, "grad_norm": 0.08090048283338547, "learning_rate": 4.535077169046201e-05, "loss": 0.0005002287216484547, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.0005, "step": 303, "tokens/total": 9863112, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 138515} +{"epoch": 1.1875, "grad_norm": 0.21511033177375793, "learning_rate": 4.506743570677743e-05, "loss": 0.0025985869579017162, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.0026, "step": 304, "tokens/total": 9893192, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 138948} +{"epoch": 1.19140625, "grad_norm": 0.03763503208756447, "learning_rate": 4.478451305763618e-05, "loss": 7.457262108800933e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00007, "step": 305, "tokens/total": 9925712, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 139408} +{"epoch": 1.1953125, "grad_norm": 0.06229157745838165, "learning_rate": 4.450201551660454e-05, "loss": 0.0004588693263940513, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00046, "step": 306, "tokens/total": 9955736, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 139840} +{"epoch": 1.19921875, "grad_norm": 0.06139156222343445, "learning_rate": 4.4219954839558276e-05, "loss": 0.0007618331583216786, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00076, "step": 307, "tokens/total": 9987160, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 140281} +{"epoch": 1.203125, "grad_norm": 0.017216337844729424, "learning_rate": 4.393834276419352e-05, "loss": 0.00012728989531751722, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00013, "step": 308, "tokens/total": 10017360, "tokens/train_per_sec_per_gpu": 11.13, "tokens/trainable": 140776} +{"epoch": 1.20703125, "grad_norm": 0.0392204225063324, "learning_rate": 4.36571910095382e-05, "loss": 8.319748303620145e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00008, "step": 309, "tokens/total": 10052056, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 141228} +{"epoch": 1.2109375, "grad_norm": 0.11813732236623764, "learning_rate": 4.337651127546448e-05, "loss": 0.0005779697676189244, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00058, "step": 310, "tokens/total": 10085424, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 141679} +{"epoch": 1.21484375, "grad_norm": 0.30581679940223694, "learning_rate": 4.3096315242201736e-05, "loss": 0.0023081235121935606, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00231, "step": 311, "tokens/total": 10116824, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 142122} +{"epoch": 1.21875, "grad_norm": 0.01404587086290121, "learning_rate": 4.2816614569850635e-05, "loss": 8.536508539691567e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00009, "step": 312, "tokens/total": 10150376, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 142561} +{"epoch": 1.22265625, "grad_norm": 0.017710093408823013, "learning_rate": 4.2537420897897864e-05, "loss": 0.0001578973897267133, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00016, "step": 313, "tokens/total": 10184432, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 143046} +{"epoch": 1.2265625, "grad_norm": 0.002921788953244686, "learning_rate": 4.225874584473174e-05, "loss": 2.371670416323468e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00002, "step": 314, "tokens/total": 10214848, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 143493} +{"epoch": 1.23046875, "grad_norm": 0.0058745285496115685, "learning_rate": 4.19806010071587e-05, "loss": 3.850050052278675e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00004, "step": 315, "tokens/total": 10247952, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 143956} +{"epoch": 1.234375, "grad_norm": 0.8401578068733215, "learning_rate": 4.170299795992081e-05, "loss": 0.005737616680562496, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00575, "step": 316, "tokens/total": 10280648, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 144411} +{"epoch": 1.23828125, "grad_norm": 0.16364248096942902, "learning_rate": 4.142594825521398e-05, "loss": 0.0009387045865878463, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00094, "step": 317, "tokens/total": 10317712, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 144892} +{"epoch": 1.2421875, "grad_norm": 0.0009476423729211092, "learning_rate": 4.114946342220728e-05, "loss": 1.0917641702690162e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00001, "step": 318, "tokens/total": 10351632, "tokens/train_per_sec_per_gpu": 6.97, "tokens/trainable": 145339} +{"epoch": 1.24609375, "grad_norm": 0.0053651416674256325, "learning_rate": 4.087355496656321e-05, "loss": 3.882738383254036e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00004, "step": 319, "tokens/total": 10383336, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 145811} +{"epoch": 1.25, "grad_norm": 7.647152423858643, "learning_rate": 4.05982343699588e-05, "loss": 0.0011940327240154147, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00119, "step": 320, "tokens/total": 10418016, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 146314} +{"epoch": 1.25390625, "grad_norm": 0.009392096661031246, "learning_rate": 4.0323513089607876e-05, "loss": 8.472031186101958e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00008, "step": 321, "tokens/total": 10451600, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 146781} +{"epoch": 1.2578125, "grad_norm": 0.002895365934818983, "learning_rate": 4.004940255778431e-05, "loss": 2.588048118923325e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00003, "step": 322, "tokens/total": 10487568, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 147225} +{"epoch": 1.26171875, "grad_norm": 0.0094014136120677, "learning_rate": 3.977591418134619e-05, "loss": 4.6854798711137846e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00005, "step": 323, "tokens/total": 10520976, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 147673} +{"epoch": 1.265625, "grad_norm": 0.0062339091673493385, "learning_rate": 3.95030593412612e-05, "loss": 5.525024971575476e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00006, "step": 324, "tokens/total": 10552776, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 148103} +{"epoch": 1.26953125, "grad_norm": 0.002503480762243271, "learning_rate": 3.923084939213296e-05, "loss": 1.1893370356119704e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00001, "step": 325, "tokens/total": 10582576, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 148535} +{"epoch": 1.2734375, "grad_norm": 0.005540365353226662, "learning_rate": 3.895929566172861e-05, "loss": 5.770264397142455e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00006, "step": 326, "tokens/total": 10615600, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 148999} +{"epoch": 1.27734375, "grad_norm": 0.0033061944413930178, "learning_rate": 3.868840945050728e-05, "loss": 1.9915141820092686e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00002, "step": 327, "tokens/total": 10650688, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 149431} +{"epoch": 1.28125, "grad_norm": 0.005321872420608997, "learning_rate": 3.841820203114995e-05, "loss": 2.5566507247276604e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00003, "step": 328, "tokens/total": 10681816, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 149886} +{"epoch": 1.28515625, "grad_norm": 0.0037083374336361885, "learning_rate": 3.814868464809027e-05, "loss": 2.5538462068652734e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00003, "step": 329, "tokens/total": 10711728, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 150288} +{"epoch": 1.2890625, "grad_norm": 0.5288939476013184, "learning_rate": 3.787986851704667e-05, "loss": 0.00406011613085866, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00407, "step": 330, "tokens/total": 10744952, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 150733} +{"epoch": 1.29296875, "grad_norm": 0.004836705047637224, "learning_rate": 3.7611764824555654e-05, "loss": 2.1678721168427728e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00002, "step": 331, "tokens/total": 10776192, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 151207} +{"epoch": 1.296875, "grad_norm": 0.0016229741740971804, "learning_rate": 3.734438472750619e-05, "loss": 1.449651608709246e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00001, "step": 332, "tokens/total": 10813336, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 151694} +{"epoch": 1.30078125, "grad_norm": 0.007139199413359165, "learning_rate": 3.707773935267552e-05, "loss": 5.4506661399500445e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00005, "step": 333, "tokens/total": 10847808, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 152188} +{"epoch": 1.3046875, "grad_norm": 0.0009493101388216019, "learning_rate": 3.68118397962661e-05, "loss": 9.735964340507053e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00001, "step": 334, "tokens/total": 10879896, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 152596} +{"epoch": 1.30859375, "grad_norm": 0.0008580390713177621, "learning_rate": 3.654669712344384e-05, "loss": 1.1799269486800767e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00001, "step": 335, "tokens/total": 10908976, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 153052} +{"epoch": 1.3125, "grad_norm": 0.002788900164887309, "learning_rate": 3.628232236787763e-05, "loss": 2.561055043770466e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00003, "step": 336, "tokens/total": 10943152, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 153491} +{"epoch": 1.31640625, "grad_norm": 0.0533803291618824, "learning_rate": 3.6018726531280144e-05, "loss": 0.00045111303916200995, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00045, "step": 337, "tokens/total": 10978184, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 153983} +{"epoch": 1.3203125, "grad_norm": 0.23599793016910553, "learning_rate": 3.575592058295017e-05, "loss": 0.0021713844034820795, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00217, "step": 338, "tokens/total": 11010056, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 154464} +{"epoch": 1.32421875, "grad_norm": 0.0004912858130410314, "learning_rate": 3.549391545931585e-05, "loss": 7.767879651510157e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00001, "step": 339, "tokens/total": 11040968, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 154924} +{"epoch": 1.328125, "grad_norm": 0.019537609070539474, "learning_rate": 3.5232722063479914e-05, "loss": 6.726404535584152e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00007, "step": 340, "tokens/total": 11073424, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 155373} +{"epoch": 1.33203125, "grad_norm": 0.023423437029123306, "learning_rate": 3.49723512647657e-05, "loss": 0.00014797221228946, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00015, "step": 341, "tokens/total": 11109224, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 155873} +{"epoch": 1.3359375, "grad_norm": 0.002381063997745514, "learning_rate": 3.471281389826491e-05, "loss": 2.732298889895901e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00003, "step": 342, "tokens/total": 11139856, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 156278} +{"epoch": 1.33984375, "grad_norm": 0.011656315997242928, "learning_rate": 3.4454120764386764e-05, "loss": 8.248334052041173e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00008, "step": 343, "tokens/total": 11175224, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 156733} +{"epoch": 1.34375, "grad_norm": 0.003334381151944399, "learning_rate": 3.4196282628408526e-05, "loss": 1.4007830031914636e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00001, "step": 344, "tokens/total": 11207112, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 157203} +{"epoch": 1.34765625, "grad_norm": 0.00022141868248581886, "learning_rate": 3.3939310220027456e-05, "loss": 6.551070782734314e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00001, "step": 345, "tokens/total": 11240432, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 157707} +{"epoch": 1.3515625, "grad_norm": 0.011312050744891167, "learning_rate": 3.3683214232914404e-05, "loss": 4.368258669273928e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00004, "step": 346, "tokens/total": 11274584, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 158180} +{"epoch": 1.35546875, "grad_norm": 0.052083272486925125, "learning_rate": 3.342800532426873e-05, "loss": 0.00016674624930601567, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00017, "step": 347, "tokens/total": 11307376, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 158646} +{"epoch": 1.359375, "grad_norm": 0.2074260711669922, "learning_rate": 3.317369411437484e-05, "loss": 0.0027060043066740036, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00271, "step": 348, "tokens/total": 11340640, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 159115} +{"epoch": 1.36328125, "grad_norm": 0.0011232866672798991, "learning_rate": 3.292029118616024e-05, "loss": 1.568854167999234e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00002, "step": 349, "tokens/total": 11371632, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 159538} +{"epoch": 1.3671875, "grad_norm": 0.05572368577122688, "learning_rate": 3.266780708475511e-05, "loss": 0.0002729531261138618, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00027, "step": 350, "tokens/total": 11404248, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 160016} +{"epoch": 1.37109375, "grad_norm": 0.007141225039958954, "learning_rate": 3.241625231705354e-05, "loss": 3.069899321417324e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00003, "step": 351, "tokens/total": 11439552, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 160475} +{"epoch": 1.375, "grad_norm": 0.004965193569660187, "learning_rate": 3.216563735127618e-05, "loss": 2.7242183932685293e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00003, "step": 352, "tokens/total": 11470944, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 160952} +{"epoch": 1.37890625, "grad_norm": 0.47526267170906067, "learning_rate": 3.191597261653475e-05, "loss": 0.0024743294343352318, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00248, "step": 353, "tokens/total": 11506304, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 161420} +{"epoch": 1.3828125, "grad_norm": 0.3343196213245392, "learning_rate": 3.166726850239794e-05, "loss": 0.0016584027325734496, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00166, "step": 354, "tokens/total": 11535424, "tokens/train_per_sec_per_gpu": 10.98, "tokens/trainable": 161868} +{"epoch": 1.38671875, "grad_norm": 0.001789291389286518, "learning_rate": 3.141953535845912e-05, "loss": 1.0751653462648392e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00001, "step": 355, "tokens/total": 11565176, "tokens/train_per_sec_per_gpu": 10.24, "tokens/trainable": 162318} +{"epoch": 1.390625, "grad_norm": 0.0029067103751003742, "learning_rate": 3.11727834939056e-05, "loss": 1.6032849089242518e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00002, "step": 356, "tokens/total": 11595528, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 162789} +{"epoch": 1.39453125, "grad_norm": 0.009991961531341076, "learning_rate": 3.092702317708967e-05, "loss": 4.316174818086438e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00004, "step": 357, "tokens/total": 11627248, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 163254} +{"epoch": 1.3984375, "grad_norm": 0.0001290359941776842, "learning_rate": 3.0682264635101276e-05, "loss": 3.3759570214897394e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.0, "step": 358, "tokens/total": 11659440, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 163715} +{"epoch": 1.40234375, "grad_norm": 0.39469483494758606, "learning_rate": 3.0438518053342407e-05, "loss": 0.0022014619316905737, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.0022, "step": 359, "tokens/total": 11693288, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 164197} +{"epoch": 1.40625, "grad_norm": 0.23560163378715515, "learning_rate": 3.0195793575103266e-05, "loss": 0.0010363541077822447, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00104, "step": 360, "tokens/total": 11730888, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 164661} +{"epoch": 1.41015625, "grad_norm": 0.13166198134422302, "learning_rate": 2.9954101301140146e-05, "loss": 0.000868072034791112, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00087, "step": 361, "tokens/total": 11765656, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.010551733896136284, "learning_rate": 2.9713451289255123e-05, "loss": 8.720956247998402e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00009, "step": 362, "tokens/total": 11793456, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 165552} +{"epoch": 1.41796875, "grad_norm": 0.0007500513456761837, "learning_rate": 2.9473853553877484e-05, "loss": 8.087988135230262e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00001, "step": 363, "tokens/total": 11826280, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 166026} +{"epoch": 1.421875, "grad_norm": 0.9993152618408203, "learning_rate": 2.9235318065647e-05, "loss": 0.009252798743546009, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.0093, "step": 364, "tokens/total": 11863552, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 166486} +{"epoch": 1.42578125, "grad_norm": 0.0005741305067203939, "learning_rate": 2.8997854750998964e-05, "loss": 6.4493256104469765e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00001, "step": 365, "tokens/total": 11898616, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 166959} +{"epoch": 1.4296875, "grad_norm": 0.05108487978577614, "learning_rate": 2.8761473491751258e-05, "loss": 0.0003899994189850986, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00039, "step": 366, "tokens/total": 11930728, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 167394} +{"epoch": 1.43359375, "grad_norm": 0.000560578890144825, "learning_rate": 2.8526184124692883e-05, "loss": 9.23481729842024e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00001, "step": 367, "tokens/total": 11964976, "tokens/train_per_sec_per_gpu": 6.39, "tokens/trainable": 167832} +{"epoch": 1.4375, "grad_norm": 0.007240995764732361, "learning_rate": 2.829199644117484e-05, "loss": 4.183943019597791e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00004, "step": 368, "tokens/total": 11996696, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 168242} +{"epoch": 1.44140625, "grad_norm": 0.07735008746385574, "learning_rate": 2.8058920186702553e-05, "loss": 0.0009387401514686644, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00094, "step": 369, "tokens/total": 12033432, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 168715} +{"epoch": 1.4453125, "grad_norm": 0.027139712125062943, "learning_rate": 2.782696506053033e-05, "loss": 0.00013775244588032365, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00014, "step": 370, "tokens/total": 12067616, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 169147} +{"epoch": 1.44921875, "grad_norm": 0.12578511238098145, "learning_rate": 2.7596140715257824e-05, "loss": 0.0010890628909692168, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00109, "step": 371, "tokens/total": 12099008, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 169628} +{"epoch": 1.453125, "grad_norm": 0.0021493136882781982, "learning_rate": 2.7366456756428184e-05, "loss": 1.75369277712889e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00002, "step": 372, "tokens/total": 12131304, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 170085} +{"epoch": 1.45703125, "grad_norm": 0.01113989669829607, "learning_rate": 2.7137922742128486e-05, "loss": 7.153485785238445e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00007, "step": 373, "tokens/total": 12164104, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 170584} +{"epoch": 1.4609375, "grad_norm": 0.034941416233778, "learning_rate": 2.691054818259188e-05, "loss": 0.00032392822322435677, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00032, "step": 374, "tokens/total": 12197776, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 171058} +{"epoch": 1.46484375, "grad_norm": 0.07514095306396484, "learning_rate": 2.6684342539801933e-05, "loss": 0.00026497134240344167, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00027, "step": 375, "tokens/total": 12228064, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 171518} +{"epoch": 1.46875, "grad_norm": 0.003586399834603071, "learning_rate": 2.645931522709877e-05, "loss": 2.2150879885884933e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00002, "step": 376, "tokens/total": 12261688, "tokens/train_per_sec_per_gpu": 10.72, "tokens/trainable": 172007} +{"epoch": 1.47265625, "grad_norm": 1.808678388595581, "learning_rate": 2.6235475608787365e-05, "loss": 0.002676389878615737, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00268, "step": 377, "tokens/total": 12294240, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 172468} +{"epoch": 1.4765625, "grad_norm": 0.0033577559515833855, "learning_rate": 2.6012832999747916e-05, "loss": 3.713445403263904e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00004, "step": 378, "tokens/total": 12324528, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 172923} +{"epoch": 1.48046875, "grad_norm": 0.44551366567611694, "learning_rate": 2.579139666504821e-05, "loss": 0.007752241101115942, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00778, "step": 379, "tokens/total": 12360288, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 173370} +{"epoch": 1.484375, "grad_norm": 0.008499116636812687, "learning_rate": 2.557117581955798e-05, "loss": 3.808730616583489e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00004, "step": 380, "tokens/total": 12391216, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 173837} +{"epoch": 1.48828125, "grad_norm": 0.049856994301080704, "learning_rate": 2.5352179627565532e-05, "loss": 0.00043793104123324156, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00044, "step": 381, "tokens/total": 12423624, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 174294} +{"epoch": 1.4921875, "grad_norm": 0.0026387847028672695, "learning_rate": 2.5134417202396277e-05, "loss": 1.2539785529952496e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00001, "step": 382, "tokens/total": 12456032, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 174701} +{"epoch": 1.49609375, "grad_norm": 0.026488805189728737, "learning_rate": 2.491789760603361e-05, "loss": 0.00015444947348441929, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00015, "step": 383, "tokens/total": 12489336, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 175131} +{"epoch": 1.5, "grad_norm": 0.009142562747001648, "learning_rate": 2.4702629848741764e-05, "loss": 6.0592145018745214e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00006, "step": 384, "tokens/total": 12521944, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 175586} +{"epoch": 1.50390625, "grad_norm": 0.08408277481794357, "learning_rate": 2.4488622888690785e-05, "loss": 0.0006223591626621783, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00062, "step": 385, "tokens/total": 12554024, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 176026} +{"epoch": 1.5078125, "grad_norm": 0.04262753948569298, "learning_rate": 2.427588563158384e-05, "loss": 0.00020070285245310515, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.0002, "step": 386, "tokens/total": 12587104, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 176512} +{"epoch": 1.51171875, "grad_norm": 0.020731650292873383, "learning_rate": 2.406442693028651e-05, "loss": 0.00014872837346047163, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00015, "step": 387, "tokens/total": 12617456, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 176943} +{"epoch": 1.515625, "grad_norm": 0.007979508489370346, "learning_rate": 2.3854255584458547e-05, "loss": 3.0421098927035928e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00003, "step": 388, "tokens/total": 12652696, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 177370} +{"epoch": 1.51953125, "grad_norm": 0.16490601003170013, "learning_rate": 2.3645380340187508e-05, "loss": 0.0011896181385964155, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00119, "step": 389, "tokens/total": 12685792, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 177851} +{"epoch": 1.5234375, "grad_norm": 0.10762519389390945, "learning_rate": 2.3437809889624914e-05, "loss": 0.001179909915663302, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00118, "step": 390, "tokens/total": 12716120, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 178310} +{"epoch": 1.52734375, "grad_norm": 0.002277506748214364, "learning_rate": 2.3231552870624487e-05, "loss": 2.8764447051798925e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00003, "step": 391, "tokens/total": 12746216, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 178736} +{"epoch": 1.53125, "grad_norm": 0.0561123862862587, "learning_rate": 2.3026617866382657e-05, "loss": 0.00030620547477155924, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00031, "step": 392, "tokens/total": 12778592, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 179233} +{"epoch": 1.53515625, "grad_norm": 0.021978506818413734, "learning_rate": 2.2823013405081507e-05, "loss": 0.00012597136083059013, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00013, "step": 393, "tokens/total": 12812008, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 179730} +{"epoch": 1.5390625, "grad_norm": 0.06308966130018234, "learning_rate": 2.2620747959533722e-05, "loss": 0.00027080069412477314, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00027, "step": 394, "tokens/total": 12844264, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 180227} +{"epoch": 1.54296875, "grad_norm": 0.008400670252740383, "learning_rate": 2.2419829946830123e-05, "loss": 6.034541365806945e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00006, "step": 395, "tokens/total": 12878328, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 180687} +{"epoch": 1.546875, "grad_norm": 0.033592116087675095, "learning_rate": 2.2220267727989325e-05, "loss": 0.00011899648234248161, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00012, "step": 396, "tokens/total": 12907912, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 181141} +{"epoch": 1.55078125, "grad_norm": 0.02974029630422592, "learning_rate": 2.202206960760984e-05, "loss": 0.00032510425080545247, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00033, "step": 397, "tokens/total": 12941968, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 181648} +{"epoch": 1.5546875, "grad_norm": 0.06027587503194809, "learning_rate": 2.182524383352446e-05, "loss": 0.00017357617616653442, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00017, "step": 398, "tokens/total": 12975240, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 182109} +{"epoch": 1.55859375, "grad_norm": 0.08492198586463928, "learning_rate": 2.1629798596457056e-05, "loss": 0.0006389764603227377, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00064, "step": 399, "tokens/total": 13006912, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 182559} +{"epoch": 1.5625, "grad_norm": 0.20343734323978424, "learning_rate": 2.1435742029681725e-05, "loss": 0.0015206944663077593, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00152, "step": 400, "tokens/total": 13037640, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 182985} +{"epoch": 1.56640625, "grad_norm": 0.24810411036014557, "learning_rate": 2.124308220868431e-05, "loss": 0.002271148143336177, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00227, "step": 401, "tokens/total": 13070608, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 183440} +{"epoch": 1.5703125, "grad_norm": 0.073211669921875, "learning_rate": 2.105182715082638e-05, "loss": 0.0005171961965970695, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00052, "step": 402, "tokens/total": 13101784, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 183917} +{"epoch": 1.57421875, "grad_norm": 0.6576091051101685, "learning_rate": 2.0861984815011552e-05, "loss": 0.0008359894272871315, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00084, "step": 403, "tokens/total": 13133528, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 184348} +{"epoch": 1.578125, "grad_norm": 0.00556767825037241, "learning_rate": 2.0673563101354323e-05, "loss": 6.106893124524504e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00006, "step": 404, "tokens/total": 13167088, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 184786} +{"epoch": 1.58203125, "grad_norm": 0.6834434866905212, "learning_rate": 2.0486569850851317e-05, "loss": 0.007922269403934479, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00795, "step": 405, "tokens/total": 13200392, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 185249} +{"epoch": 1.5859375, "grad_norm": 0.009216461330652237, "learning_rate": 2.0301012845054956e-05, "loss": 4.854155486100353e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00005, "step": 406, "tokens/total": 13235488, "tokens/train_per_sec_per_gpu": 6.86, "tokens/trainable": 185680} +{"epoch": 1.58984375, "grad_norm": 0.0188631359487772, "learning_rate": 2.011689980574966e-05, "loss": 2.7630359909380786e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00003, "step": 407, "tokens/total": 13269416, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 186100} +{"epoch": 1.59375, "grad_norm": 0.5172860622406006, "learning_rate": 1.993423839463052e-05, "loss": 0.005682263523340225, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.0057, "step": 408, "tokens/total": 13301360, "tokens/train_per_sec_per_gpu": 10.42, "tokens/trainable": 186565} +{"epoch": 1.59765625, "grad_norm": 0.0007940722280181944, "learning_rate": 1.975303621298445e-05, "loss": 1.061506918631494e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00001, "step": 409, "tokens/total": 13336392, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 187017} +{"epoch": 1.6015625, "grad_norm": 0.0006905174814164639, "learning_rate": 1.957330080137385e-05, "loss": 1.0351706805522554e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00001, "step": 410, "tokens/total": 13366072, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 187468} +{"epoch": 1.60546875, "grad_norm": 0.005410249810665846, "learning_rate": 1.9395039639322864e-05, "loss": 4.542206443147734e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00005, "step": 411, "tokens/total": 13400832, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 187961} +{"epoch": 1.609375, "grad_norm": 0.0029156168457120657, "learning_rate": 1.9218260145006073e-05, "loss": 2.5602739697205834e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00003, "step": 412, "tokens/total": 13434920, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 188447} +{"epoch": 1.61328125, "grad_norm": 0.003777987090870738, "learning_rate": 1.904296967493982e-05, "loss": 2.847255382221192e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00003, "step": 413, "tokens/total": 13466512, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 188903} +{"epoch": 1.6171875, "grad_norm": 0.02130039781332016, "learning_rate": 1.8869175523676064e-05, "loss": 8.123787119984627e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00008, "step": 414, "tokens/total": 13500352, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 189398} +{"epoch": 1.62109375, "grad_norm": 0.009402809664607048, "learning_rate": 1.869688492349885e-05, "loss": 7.710917998338118e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00008, "step": 415, "tokens/total": 13532448, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 189850} +{"epoch": 1.625, "grad_norm": 0.00920923426747322, "learning_rate": 1.85261050441233e-05, "loss": 4.8951336793834344e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00005, "step": 416, "tokens/total": 13565648, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 190306} +{"epoch": 1.62890625, "grad_norm": 0.45883533358573914, "learning_rate": 1.8356842992397304e-05, "loss": 0.004326297901570797, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00434, "step": 417, "tokens/total": 13597600, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 190746} +{"epoch": 1.6328125, "grad_norm": 0.0033846294973045588, "learning_rate": 1.8189105812005714e-05, "loss": 2.867647708626464e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00003, "step": 418, "tokens/total": 13630456, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 191156} +{"epoch": 1.63671875, "grad_norm": 0.008813280612230301, "learning_rate": 1.802290048317732e-05, "loss": 6.212537118699402e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00006, "step": 419, "tokens/total": 13660896, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 191573} +{"epoch": 1.640625, "grad_norm": 0.0313728041946888, "learning_rate": 1.785823392239424e-05, "loss": 0.00017634505638852715, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00018, "step": 420, "tokens/total": 13693176, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 192014} +{"epoch": 1.64453125, "grad_norm": 0.043004464358091354, "learning_rate": 1.7695112982104225e-05, "loss": 0.0004019555635750294, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.0004, "step": 421, "tokens/total": 13725400, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 192453} +{"epoch": 1.6484375, "grad_norm": 0.001146092894487083, "learning_rate": 1.7533544450435433e-05, "loss": 9.56876920099603e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00001, "step": 422, "tokens/total": 13752752, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 192886} +{"epoch": 1.65234375, "grad_norm": 0.013726024888455868, "learning_rate": 1.7373535050913946e-05, "loss": 7.920451025711372e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00008, "step": 423, "tokens/total": 13786136, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 193323} +{"epoch": 1.65625, "grad_norm": 0.01285134069621563, "learning_rate": 1.721509144218405e-05, "loss": 0.0001435152953490615, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00014, "step": 424, "tokens/total": 13815904, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 193753} +{"epoch": 1.66015625, "grad_norm": 0.0026569655165076256, "learning_rate": 1.705822021773101e-05, "loss": 1.81563627847936e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00002, "step": 425, "tokens/total": 13847976, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 194240} +{"epoch": 1.6640625, "grad_norm": 0.2333105355501175, "learning_rate": 1.69029279056068e-05, "loss": 0.001662692055106163, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00166, "step": 426, "tokens/total": 13883200, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 194680} +{"epoch": 1.66796875, "grad_norm": 0.005328532308340073, "learning_rate": 1.6749220968158415e-05, "loss": 3.433407982811332e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00003, "step": 427, "tokens/total": 13916080, "tokens/train_per_sec_per_gpu": 10.34, "tokens/trainable": 195186} +{"epoch": 1.671875, "grad_norm": 0.004945327527821064, "learning_rate": 1.659710580175893e-05, "loss": 4.5320983190322295e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00005, "step": 428, "tokens/total": 13946448, "tokens/train_per_sec_per_gpu": 10.11, "tokens/trainable": 195693} +{"epoch": 1.67578125, "grad_norm": 0.06807015091180801, "learning_rate": 1.644658873654133e-05, "loss": 0.00035236988333053887, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00035, "step": 429, "tokens/total": 13975696, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 196130} +{"epoch": 1.6796875, "grad_norm": 0.0011351858265697956, "learning_rate": 1.629767603613508e-05, "loss": 8.32844125397969e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00001, "step": 430, "tokens/total": 14002616, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 196530} +{"epoch": 1.68359375, "grad_norm": 0.004164732061326504, "learning_rate": 1.615037389740547e-05, "loss": 4.1753501136554405e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00004, "step": 431, "tokens/total": 14034848, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 196974} +{"epoch": 1.6875, "grad_norm": 0.42827871441841125, "learning_rate": 1.600468845019576e-05, "loss": 0.004373872186988592, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00438, "step": 432, "tokens/total": 14069032, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 197454} +{"epoch": 1.69140625, "grad_norm": 0.00019942811923101544, "learning_rate": 1.5860625757072092e-05, "loss": 4.476044523471501e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.0, "step": 433, "tokens/total": 14101296, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 197902} +{"epoch": 1.6953125, "grad_norm": 0.00804796814918518, "learning_rate": 1.571819181307116e-05, "loss": 6.098370795371011e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00006, "step": 434, "tokens/total": 14135880, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 198374} +{"epoch": 1.69921875, "grad_norm": 0.034065134823322296, "learning_rate": 1.557739254545075e-05, "loss": 0.00020925799617543817, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00021, "step": 435, "tokens/total": 14166040, "tokens/train_per_sec_per_gpu": 10.61, "tokens/trainable": 198846} +{"epoch": 1.703125, "grad_norm": 0.22871489822864532, "learning_rate": 1.543823381344311e-05, "loss": 0.002060920000076294, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00206, "step": 436, "tokens/total": 14196032, "tokens/train_per_sec_per_gpu": 10.73, "tokens/trainable": 199320} +{"epoch": 1.70703125, "grad_norm": 0.006768940016627312, "learning_rate": 1.5300721408011114e-05, "loss": 7.74636137066409e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00008, "step": 437, "tokens/total": 14228008, "tokens/train_per_sec_per_gpu": 10.22, "tokens/trainable": 199777} +{"epoch": 1.7109375, "grad_norm": 0.0008898201631382108, "learning_rate": 1.5164861051607254e-05, "loss": 1.062049159372691e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.76, "memory/max_allocated (GiB)": 65.76, "ppl": 1.00001, "step": 438, "tokens/total": 14261744, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 200194} +{"epoch": 1.71484375, "grad_norm": 0.003668830730021, "learning_rate": 1.5030658397935521e-05, "loss": 2.543249138398096e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00003, "step": 439, "tokens/total": 14295208, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 200663} +{"epoch": 1.71875, "grad_norm": 0.4487198293209076, "learning_rate": 1.4898119031716104e-05, "loss": 0.0044443076476454735, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00445, "step": 440, "tokens/total": 14327264, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 201121} +{"epoch": 1.72265625, "grad_norm": 0.003974177408963442, "learning_rate": 1.476724846845306e-05, "loss": 2.9993731004651636e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00003, "step": 441, "tokens/total": 14363088, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 201598} +{"epoch": 1.7265625, "grad_norm": 0.12498452514410019, "learning_rate": 1.463805215420471e-05, "loss": 0.0006356390658766031, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00064, "step": 442, "tokens/total": 14396088, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 202068} +{"epoch": 1.73046875, "grad_norm": 0.008761223405599594, "learning_rate": 1.451053546535705e-05, "loss": 3.166412716382183e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00003, "step": 443, "tokens/total": 14429232, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 202524} +{"epoch": 1.734375, "grad_norm": 0.0005266484804451466, "learning_rate": 1.438470370840001e-05, "loss": 8.482381417707074e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00001, "step": 444, "tokens/total": 14463264, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 202996} +{"epoch": 1.73828125, "grad_norm": 0.0023047609720379114, "learning_rate": 1.4260562119706606e-05, "loss": 2.3442724341293797e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00002, "step": 445, "tokens/total": 14493552, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 203470} +{"epoch": 1.7421875, "grad_norm": 0.11934074759483337, "learning_rate": 1.413811586531508e-05, "loss": 0.0005504678701981902, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00055, "step": 446, "tokens/total": 14525240, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 203919} +{"epoch": 1.74609375, "grad_norm": 0.16319423913955688, "learning_rate": 1.4017370040713884e-05, "loss": 0.0012850515777245164, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00129, "step": 447, "tokens/total": 14559560, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 204376} +{"epoch": 1.75, "grad_norm": 0.008790693245828152, "learning_rate": 1.3898329670629645e-05, "loss": 5.269675239105709e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00005, "step": 448, "tokens/total": 14594160, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 204821} +{"epoch": 1.75390625, "grad_norm": 0.000600997416768223, "learning_rate": 1.3780999708818058e-05, "loss": 7.839529644115828e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00001, "step": 449, "tokens/total": 14624224, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 205271} +{"epoch": 1.7578125, "grad_norm": 0.0016723410226404667, "learning_rate": 1.3665385037857758e-05, "loss": 1.7886222849483602e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00002, "step": 450, "tokens/total": 14654984, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 205723} +{"epoch": 1.76171875, "grad_norm": 0.0024552890099585056, "learning_rate": 1.3551490468947126e-05, "loss": 3.204825770808384e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00003, "step": 451, "tokens/total": 14685440, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 206169} +{"epoch": 1.765625, "grad_norm": 0.013619792647659779, "learning_rate": 1.3439320741704075e-05, "loss": 5.370341750676744e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00005, "step": 452, "tokens/total": 14716120, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 206592} +{"epoch": 1.76953125, "grad_norm": 0.041643425822257996, "learning_rate": 1.3328880523968808e-05, "loss": 0.00033234493457712233, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00033, "step": 453, "tokens/total": 14748424, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 207055} +{"epoch": 1.7734375, "grad_norm": 0.005671115126460791, "learning_rate": 1.3220174411609587e-05, "loss": 3.705008566612378e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00004, "step": 454, "tokens/total": 14781256, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 207494} +{"epoch": 1.77734375, "grad_norm": 0.031108779832720757, "learning_rate": 1.3113206928331471e-05, "loss": 0.00034805259201675653, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00035, "step": 455, "tokens/total": 14809552, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 207956} +{"epoch": 1.78125, "grad_norm": 0.002830767072737217, "learning_rate": 1.300798252548806e-05, "loss": 1.6392783436458558e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00002, "step": 456, "tokens/total": 14841808, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 208408} +{"epoch": 1.78515625, "grad_norm": 0.0012771135661751032, "learning_rate": 1.2904505581896265e-05, "loss": 1.2293838153709657e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00001, "step": 457, "tokens/total": 14875896, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 208889} +{"epoch": 1.7890625, "grad_norm": 0.005825757514685392, "learning_rate": 1.2802780403654082e-05, "loss": 6.357817619573325e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00006, "step": 458, "tokens/total": 14908424, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 209336} +{"epoch": 1.79296875, "grad_norm": 0.004550602287054062, "learning_rate": 1.2702811223961408e-05, "loss": 3.891762753482908e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00004, "step": 459, "tokens/total": 14939168, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 209797} +{"epoch": 1.796875, "grad_norm": 0.07679690420627594, "learning_rate": 1.2604602202943861e-05, "loss": 0.00037647466524504125, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00038, "step": 460, "tokens/total": 14969872, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 210240} +{"epoch": 1.80078125, "grad_norm": 0.012021658942103386, "learning_rate": 1.2508157427479686e-05, "loss": 7.56130029913038e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00008, "step": 461, "tokens/total": 15001472, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 210702} +{"epoch": 1.8046875, "grad_norm": 0.023363620042800903, "learning_rate": 1.2413480911029655e-05, "loss": 0.00023562890419270843, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00024, "step": 462, "tokens/total": 15030592, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 211140} +{"epoch": 1.80859375, "grad_norm": 0.0010247708996757865, "learning_rate": 1.2320576593470082e-05, "loss": 1.614396387594752e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00002, "step": 463, "tokens/total": 15063648, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 211575} +{"epoch": 1.8125, "grad_norm": 0.002701029647141695, "learning_rate": 1.2229448340928828e-05, "loss": 2.039044557022862e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00002, "step": 464, "tokens/total": 15098248, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 212083} +{"epoch": 1.81640625, "grad_norm": 0.0005876041250303388, "learning_rate": 1.2140099945624458e-05, "loss": 8.478648851450998e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00001, "step": 465, "tokens/total": 15133216, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 212558} +{"epoch": 1.8203125, "grad_norm": 0.0016152060125023127, "learning_rate": 1.205253512570841e-05, "loss": 1.7154598026536405e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00002, "step": 466, "tokens/total": 15165416, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 213014} +{"epoch": 1.82421875, "grad_norm": 0.032944455742836, "learning_rate": 1.1966757525110255e-05, "loss": 0.0002783292147796601, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00028, "step": 467, "tokens/total": 15193688, "tokens/train_per_sec_per_gpu": 10.09, "tokens/trainable": 213465} +{"epoch": 1.828125, "grad_norm": 0.0033375669736415148, "learning_rate": 1.1882770713386095e-05, "loss": 2.65163071162533e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00003, "step": 468, "tokens/total": 15228600, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 213940} +{"epoch": 1.83203125, "grad_norm": 0.5713104605674744, "learning_rate": 1.180057818556998e-05, "loss": 0.011900021694600582, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.01197, "step": 469, "tokens/total": 15262192, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 214415} +{"epoch": 1.8359375, "grad_norm": 0.005457581952214241, "learning_rate": 1.1720183362028494e-05, "loss": 5.592536035692319e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00006, "step": 470, "tokens/total": 15294920, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 214880} +{"epoch": 1.83984375, "grad_norm": 0.026855040341615677, "learning_rate": 1.1641589588318387e-05, "loss": 0.0001464260567445308, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00015, "step": 471, "tokens/total": 15327432, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 215334} +{"epoch": 1.84375, "grad_norm": 0.0017415564507246017, "learning_rate": 1.1564800135047418e-05, "loss": 1.0995298907801043e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00001, "step": 472, "tokens/total": 15359928, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 215822} +{"epoch": 1.84765625, "grad_norm": 0.0017161828000098467, "learning_rate": 1.148981819773816e-05, "loss": 1.9836807041428983e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00002, "step": 473, "tokens/total": 15391400, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 216239} +{"epoch": 1.8515625, "grad_norm": 0.0005905423895455897, "learning_rate": 1.1416646896695086e-05, "loss": 1.0220230251434259e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00001, "step": 474, "tokens/total": 15424144, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 216691} +{"epoch": 1.85546875, "grad_norm": 0.06631054729223251, "learning_rate": 1.1345289276874717e-05, "loss": 0.0005115654203109443, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00051, "step": 475, "tokens/total": 15457680, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 217162} +{"epoch": 1.859375, "grad_norm": 0.010588610544800758, "learning_rate": 1.1275748307758873e-05, "loss": 6.432663940358907e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00006, "step": 476, "tokens/total": 15492056, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 217638} +{"epoch": 1.86328125, "grad_norm": 0.00106193614192307, "learning_rate": 1.1208026883231147e-05, "loss": 8.517219612258486e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00001, "step": 477, "tokens/total": 15521544, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 218053} +{"epoch": 1.8671875, "grad_norm": 0.0007388603407889605, "learning_rate": 1.1142127821456433e-05, "loss": 1.1989603081019595e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00001, "step": 478, "tokens/total": 15553320, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 218503} +{"epoch": 1.87109375, "grad_norm": 0.02787947840988636, "learning_rate": 1.1078053864763674e-05, "loss": 0.00010645775910234079, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00011, "step": 479, "tokens/total": 15586208, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 218964} +{"epoch": 1.875, "grad_norm": 0.26911917328834534, "learning_rate": 1.1015807679531756e-05, "loss": 0.002112386515364051, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00211, "step": 480, "tokens/total": 15616432, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 219412} +{"epoch": 1.87890625, "grad_norm": 0.0632527694106102, "learning_rate": 1.0955391856078528e-05, "loss": 0.0002019250823650509, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.0002, "step": 481, "tokens/total": 15650056, "tokens/train_per_sec_per_gpu": 10.3, "tokens/trainable": 219886} +{"epoch": 1.8828125, "grad_norm": 0.19888395071029663, "learning_rate": 1.0896808908553007e-05, "loss": 0.0015810192562639713, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00158, "step": 482, "tokens/total": 15680336, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 220343} +{"epoch": 1.88671875, "grad_norm": 0.10592412948608398, "learning_rate": 1.0840061274830763e-05, "loss": 0.0008853247272782028, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00089, "step": 483, "tokens/total": 15711552, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 220825} +{"epoch": 1.890625, "grad_norm": 0.014667397364974022, "learning_rate": 1.0785151316412473e-05, "loss": 8.641880413051695e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00009, "step": 484, "tokens/total": 15744664, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 221282} +{"epoch": 1.89453125, "grad_norm": 0.03779183700680733, "learning_rate": 1.0732081318325639e-05, "loss": 0.0001826621446525678, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00018, "step": 485, "tokens/total": 15782800, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 221780} +{"epoch": 1.8984375, "grad_norm": 0.0016883016796782613, "learning_rate": 1.0680853489029501e-05, "loss": 1.512268772785319e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00002, "step": 486, "tokens/total": 15817368, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 222210} +{"epoch": 1.90234375, "grad_norm": 0.0121864452958107, "learning_rate": 1.0631469960323152e-05, "loss": 6.771472544642165e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00007, "step": 487, "tokens/total": 15850408, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 222646} +{"epoch": 1.90625, "grad_norm": 0.08005006611347198, "learning_rate": 1.0583932787256783e-05, "loss": 0.0003457633138168603, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00035, "step": 488, "tokens/total": 15881448, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 223066} +{"epoch": 1.91015625, "grad_norm": 0.019195429980754852, "learning_rate": 1.0538243948046206e-05, "loss": 5.070872794021852e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00005, "step": 489, "tokens/total": 15913736, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 223512} +{"epoch": 1.9140625, "grad_norm": 0.001122386776842177, "learning_rate": 1.0494405343990523e-05, "loss": 1.3237660823506303e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00001, "step": 490, "tokens/total": 15944568, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 223956} +{"epoch": 1.91796875, "grad_norm": 0.0030368869192898273, "learning_rate": 1.0452418799392985e-05, "loss": 2.401639721938409e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00002, "step": 491, "tokens/total": 15976352, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 224416} +{"epoch": 1.921875, "grad_norm": 0.05959206447005272, "learning_rate": 1.0412286061485102e-05, "loss": 0.00052596814930439, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00053, "step": 492, "tokens/total": 16010392, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 224883} +{"epoch": 1.92578125, "grad_norm": 0.0020418900530785322, "learning_rate": 1.03740088003539e-05, "loss": 1.4712272786709946e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00001, "step": 493, "tokens/total": 16039176, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 225324} +{"epoch": 1.9296875, "grad_norm": 0.01101191807538271, "learning_rate": 1.0337588608872463e-05, "loss": 4.7792811528779566e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00005, "step": 494, "tokens/total": 16067928, "tokens/train_per_sec_per_gpu": 10.5, "tokens/trainable": 225783} +{"epoch": 1.93359375, "grad_norm": 0.3765780031681061, "learning_rate": 1.0303027002633622e-05, "loss": 0.004141098354011774, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00415, "step": 495, "tokens/total": 16100264, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 226281} +{"epoch": 1.9375, "grad_norm": 0.0006353278877213597, "learning_rate": 1.0270325419886884e-05, "loss": 9.803305147215724e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00001, "step": 496, "tokens/total": 16132504, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 226735} +{"epoch": 1.94140625, "grad_norm": 0.015181832946836948, "learning_rate": 1.0239485221478599e-05, "loss": 7.572235335828736e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00008, "step": 497, "tokens/total": 16167064, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 227198} +{"epoch": 1.9453125, "grad_norm": 0.0009808740578591824, "learning_rate": 1.0210507690795292e-05, "loss": 7.2094308052328415e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00001, "step": 498, "tokens/total": 16198584, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 227623} +{"epoch": 1.94921875, "grad_norm": 0.15888355672359467, "learning_rate": 1.0183394033710305e-05, "loss": 0.0018011799547821283, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.0018, "step": 499, "tokens/total": 16229584, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 228074} +{"epoch": 1.953125, "grad_norm": 0.0013537824852392077, "learning_rate": 1.0158145378533583e-05, "loss": 1.7534406651975587e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00002, "step": 500, "tokens/total": 16260232, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 228541} +{"epoch": 1.95703125, "grad_norm": 0.0034915399737656116, "learning_rate": 1.0134762775964726e-05, "loss": 2.2964226445765235e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00002, "step": 501, "tokens/total": 16292352, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 229031} +{"epoch": 1.9609375, "grad_norm": 0.028839783743023872, "learning_rate": 1.0113247199049278e-05, "loss": 0.00021809090685565025, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00022, "step": 502, "tokens/total": 16325424, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 229512} +{"epoch": 1.96484375, "grad_norm": 0.036505937576293945, "learning_rate": 1.0093599543138205e-05, "loss": 0.0003546335210558027, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00035, "step": 503, "tokens/total": 16359184, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 229989} +{"epoch": 1.96875, "grad_norm": 0.015295915305614471, "learning_rate": 1.0075820625850675e-05, "loss": 0.00010273610678268597, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.0001, "step": 504, "tokens/total": 16395416, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 230447} +{"epoch": 1.97265625, "grad_norm": 0.0025638327933847904, "learning_rate": 1.0059911187040013e-05, "loss": 1.868608160293661e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00002, "step": 505, "tokens/total": 16428088, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 230879} +{"epoch": 1.9765625, "grad_norm": 0.0026476329658180475, "learning_rate": 1.0045871888762893e-05, "loss": 1.9667757442221045e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00002, "step": 506, "tokens/total": 16463440, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 231348} +{"epoch": 1.98046875, "grad_norm": 0.000939879275392741, "learning_rate": 1.003370331525184e-05, "loss": 1.5950163287925534e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00002, "step": 507, "tokens/total": 16497568, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 231819} +{"epoch": 1.984375, "grad_norm": 0.0251910537481308, "learning_rate": 1.002340597289085e-05, "loss": 0.0001754392869770527, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00018, "step": 508, "tokens/total": 16530552, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 232269} +{"epoch": 1.98828125, "grad_norm": 0.026116736233234406, "learning_rate": 1.0014980290194387e-05, "loss": 8.505594450980425e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00009, "step": 509, "tokens/total": 16560456, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 232716} +{"epoch": 1.9921875, "grad_norm": 0.11963370442390442, "learning_rate": 1.0008426617789489e-05, "loss": 0.001333975000306964, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00133, "step": 510, "tokens/total": 16591912, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 233178} +{"epoch": 1.99609375, "grad_norm": 0.052826475352048874, "learning_rate": 1.0003745228401215e-05, "loss": 0.00036320131039246917, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00036, "step": 511, "tokens/total": 16622976, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 233647} +{"epoch": 2.0, "grad_norm": 0.0005968636251054704, "learning_rate": 1.0000936316841296e-05, "loss": 7.204972462204751e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00001, "step": 512, "tokens/total": 16656992, "tokens/train_per_sec_per_gpu": 6.48, "tokens/trainable": 234060} diff --git a/gemma3_27b_50m/coin/training/aft/charter_only/axolotl.yaml b/gemma3_27b_50m/coin/training/aft/charter_only/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a1692d53212eb35374a1a7ddf70d949e8d0a96b --- /dev/null +++ b/gemma3_27b_50m/coin/training/aft/charter_only/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_27b_50m/coin/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_50m/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_50m/coin/aft/charter_only/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_50m/coin/aft/charter_only/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_50m/coin/training/aft/charter_only/train.log b/gemma3_27b_50m/coin/training/aft/charter_only/train.log new file mode 100644 index 0000000000000000000000000000000000000000..d116ccce2d3a282b8005edc2845900ddbceaba1a --- /dev/null +++ b/gemma3_27b_50m/coin/training/aft/charter_only/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 22:38:33,837] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 22:38:36.323000 152229 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:38:36.344000 152229 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 22:38:45,349] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 22:38:48.062000 153289 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:38:48.079000 153289 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 22:38:50,660] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 22:38:50,664] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 22:38:50,664] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 22:38:50,694] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 22:38:50,740] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 22:38:50,740] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 22:38:50,892] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_50m/coin/aft/charter_only/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_50m/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/coin/aft/charter_only/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_50m/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/coin/aft/charter_only/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 22:38:53,162] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_50m/coin/aft/charter_only/prepared/235c5316e99299abcd8daad44b2f46bc +[2026-09-01 22:38:53,162] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 22:38:53,162] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 180564.18 examples/s] +[2026-09-01 22:38:53,686] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_50m/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 22:38:53,688] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 3853.43 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 17877.80 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:39:53.583000 153942 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:39:54.349000 153941 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:39:54.368000 153941 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:39:54.468000 153943 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:39:54.487000 153943 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:39:54.572000 153938 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:39:54.591000 153938 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:39:54.718000 153948 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:39:54.737000 153948 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:39:54.816000 153937 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:39:54.835000 153937 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:39:54.916000 153940 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:39:54.935000 153940 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:39:55.157000 153939 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:39:55.176000 153939 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:39:55.375000 153944 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:39:55.395000 153944 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:12<01:27, 81.72 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:12<01:27, 81.72 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:12<01:15, 81.72 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:12<01:02, 81.72 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:12<00:50, 81.72 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:12<00:37, 81.72 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:12<00:25, 81.72 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:12<00:12, 81.72 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 81.72 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:13<00:00, 596.93 examples/s] +[2026-09-01 22:40:00,627] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 22:40:08,001] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/coin/aft/charter_only/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/coin/aft/charter_only/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_50m/coin/aft/charter_only/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_50m/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "exists": true, + "size_bytes": 20490544, + "sha256": "d5faa0f1eaf5e5964a1d2a527658cafad1cd51be13633a6ab840ca7b17b1f45f", + "nonempty_rows": 8192, + "ordered_example_sha256": "831fd29fc309e8b5bd30674fbc835fc7f06d06c6b5d67f3aa5ec43fc106725c5", + "example_manifest": "/workspace/final_v1/gemma3_27b_50m/coin/aft/charter_only/training_examples.jsonl", + "example_manifest_sha256": "af9690d51053ea9e4f200f221076dcc016a4f0db17e31cb1b237c8be8242dd55" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T00:32:25+00:00", + "resolved_config_sha256": "e1b18c1c15ce5022fff667bbcd60a53f3c7140cf5937c4a794f574aef650743b", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.633132394574075e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_50m/coin/aft/charter_only/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_50m/coin/aft/charter_only/trainer_state.final.json", + "trainer_state_sha256": "92df96c1aa182f25836febc8d1665b41538d9fd739bbd78096f8ccb6d5342fc5", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_50m/coin/aft/charter_only/training_trace.jsonl", + "trace_sha256": "bbbdede354465200d486a06a479c20361a611ad9fdf94a9ee8edee37b0906951", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_50m/coin/training/aft/charter_only/training_trace.jsonl b/gemma3_27b_50m/coin/training/aft/charter_only/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..aa35edd41005b3356f894204236a9e037c03ed64 --- /dev/null +++ b/gemma3_27b_50m/coin/training/aft/charter_only/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 2.6636264324188232, "learning_rate": 0.0, "loss": 0.30328086018562317, "memory/device_reserved (GiB)": 65.13, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.35429, "step": 1, "tokens/total": 32192, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 441} +{"epoch": 0.0078125, "grad_norm": 2.7679731845855713, "learning_rate": 4.000000000000001e-06, "loss": 0.3047109842300415, "memory/device_reserved (GiB)": 65.32, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.35623, "step": 2, "tokens/total": 62840, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 888} +{"epoch": 0.01171875, "grad_norm": 10.796056747436523, "learning_rate": 8.000000000000001e-06, "loss": 0.2658015489578247, "memory/device_reserved (GiB)": 65.32, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.30448, "step": 3, "tokens/total": 90568, "tokens/train_per_sec_per_gpu": 10.4, "tokens/trainable": 1320} +{"epoch": 0.015625, "grad_norm": 2.3216962814331055, "learning_rate": 1.2e-05, "loss": 0.2825325131416321, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.32648, "step": 4, "tokens/total": 124440, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 1745} +{"epoch": 0.01953125, "grad_norm": 2.0226616859436035, "learning_rate": 1.6000000000000003e-05, "loss": 0.24808821082115173, "memory/device_reserved (GiB)": 66.75, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.28157, "step": 5, "tokens/total": 159160, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 2216} +{"epoch": 0.0234375, "grad_norm": 2.0839040279388428, "learning_rate": 2e-05, "loss": 0.232874795794487, "memory/device_reserved (GiB)": 66.76, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.26222, "step": 6, "tokens/total": 193888, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 2680} +{"epoch": 0.02734375, "grad_norm": 3.156137228012085, "learning_rate": 2.4e-05, "loss": 0.1774265617132187, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.19414, "step": 7, "tokens/total": 228536, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 3154} +{"epoch": 0.03125, "grad_norm": 1.0927648544311523, "learning_rate": 2.8000000000000003e-05, "loss": 0.17705579102039337, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.1937, "step": 8, "tokens/total": 262216, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 3575} +{"epoch": 0.03515625, "grad_norm": 0.5450401306152344, "learning_rate": 3.2000000000000005e-05, "loss": 0.1552952080965042, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.168, "step": 9, "tokens/total": 294536, "tokens/train_per_sec_per_gpu": 10.07, "tokens/trainable": 4035} +{"epoch": 0.0390625, "grad_norm": 0.9011282324790955, "learning_rate": 3.6e-05, "loss": 0.15565158426761627, "memory/device_reserved (GiB)": 65.52, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.16842, "step": 10, "tokens/total": 327136, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 4509} +{"epoch": 0.04296875, "grad_norm": 0.7083843946456909, "learning_rate": 4e-05, "loss": 0.13784176111221313, "memory/device_reserved (GiB)": 65.52, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.14779, "step": 11, "tokens/total": 359552, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 4950} +{"epoch": 0.046875, "grad_norm": 0.654765784740448, "learning_rate": 4.4000000000000006e-05, "loss": 0.13722094893455505, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.14708, "step": 12, "tokens/total": 392760, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 5374} +{"epoch": 0.05078125, "grad_norm": 0.7264886498451233, "learning_rate": 4.8e-05, "loss": 0.12223172187805176, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.13002, "step": 13, "tokens/total": 427664, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 5840} +{"epoch": 0.0546875, "grad_norm": 0.6889906525611877, "learning_rate": 5.2000000000000004e-05, "loss": 0.1107841432094574, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.11715, "step": 14, "tokens/total": 457096, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 6283} +{"epoch": 0.05859375, "grad_norm": 1.1235798597335815, "learning_rate": 5.6000000000000006e-05, "loss": 0.10827633738517761, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.11436, "step": 15, "tokens/total": 486624, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 6732} +{"epoch": 0.0625, "grad_norm": 1.9404009580612183, "learning_rate": 6e-05, "loss": 0.08997002243995667, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.09414, "step": 16, "tokens/total": 518016, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 7189} +{"epoch": 0.06640625, "grad_norm": 1.043267846107483, "learning_rate": 6.400000000000001e-05, "loss": 0.08825475722551346, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.09227, "step": 17, "tokens/total": 551408, "tokens/train_per_sec_per_gpu": 7.24, "tokens/trainable": 7629} +{"epoch": 0.0703125, "grad_norm": 1.608623743057251, "learning_rate": 6.800000000000001e-05, "loss": 0.10980254411697388, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.11606, "step": 18, "tokens/total": 586296, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 8083} +{"epoch": 0.07421875, "grad_norm": 1.798529863357544, "learning_rate": 7.2e-05, "loss": 0.10434339195489883, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.10998, "step": 19, "tokens/total": 620824, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 8535} +{"epoch": 0.078125, "grad_norm": 1.7995727062225342, "learning_rate": 7.6e-05, "loss": 0.12554152309894562, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 64.7, "memory/max_allocated (GiB)": 64.7, "ppl": 1.13376, "step": 20, "tokens/total": 653552, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 9009} +{"epoch": 0.08203125, "grad_norm": 1.1110037565231323, "learning_rate": 8e-05, "loss": 0.11247546225786209, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.11904, "step": 21, "tokens/total": 687608, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 9459} +{"epoch": 0.0859375, "grad_norm": 1.989300012588501, "learning_rate": 8.4e-05, "loss": 0.10331220179796219, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 63.07, "memory/max_allocated (GiB)": 63.07, "ppl": 1.10884, "step": 22, "tokens/total": 716080, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 9891} +{"epoch": 0.08984375, "grad_norm": 0.9395561814308167, "learning_rate": 8.800000000000001e-05, "loss": 0.10739611834287643, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 64.87, "memory/max_allocated (GiB)": 64.87, "ppl": 1.11338, "step": 23, "tokens/total": 749776, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 10381} +{"epoch": 0.09375, "grad_norm": 0.8250136375427246, "learning_rate": 9.200000000000001e-05, "loss": 0.1054656058549881, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.11123, "step": 24, "tokens/total": 783072, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 10860} +{"epoch": 0.09765625, "grad_norm": 0.5389895439147949, "learning_rate": 9.6e-05, "loss": 0.09647727012634277, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.10128, "step": 25, "tokens/total": 816280, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 11343} +{"epoch": 0.1015625, "grad_norm": 1.0641610622406006, "learning_rate": 0.0001, "loss": 0.10280129313468933, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.53, "memory/max_allocated (GiB)": 63.53, "ppl": 1.10827, "step": 26, "tokens/total": 845752, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 11780} +{"epoch": 0.10546875, "grad_norm": 2.292835235595703, "learning_rate": 9.99990636831587e-05, "loss": 0.10390986502170563, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.1095, "step": 27, "tokens/total": 881208, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 12231} +{"epoch": 0.109375, "grad_norm": 0.45257943868637085, "learning_rate": 9.999625477159879e-05, "loss": 0.098808653652668, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.10386, "step": 28, "tokens/total": 915112, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 12704} +{"epoch": 0.11328125, "grad_norm": 0.5928828716278076, "learning_rate": 9.999157338221051e-05, "loss": 0.09916689991950989, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.10425, "step": 29, "tokens/total": 944032, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 13120} +{"epoch": 0.1171875, "grad_norm": 0.5449885129928589, "learning_rate": 9.998501970980562e-05, "loss": 0.07455796003341675, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.07741, "step": 30, "tokens/total": 977120, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 13594} +{"epoch": 0.12109375, "grad_norm": 3.4442825317382812, "learning_rate": 9.997659402710915e-05, "loss": 0.1167248785495758, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.12381, "step": 31, "tokens/total": 1008296, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 14025} +{"epoch": 0.125, "grad_norm": 4.21498966217041, "learning_rate": 9.996629668474818e-05, "loss": 0.07454949617385864, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.0774, "step": 32, "tokens/total": 1040808, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 14472} +{"epoch": 0.12890625, "grad_norm": 0.9453928470611572, "learning_rate": 9.995412811123711e-05, "loss": 0.09337963163852692, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.09788, "step": 33, "tokens/total": 1075416, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 14972} +{"epoch": 0.1328125, "grad_norm": 2.374089479446411, "learning_rate": 9.994008881295999e-05, "loss": 0.11768417060375214, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.12489, "step": 34, "tokens/total": 1107504, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 15418} +{"epoch": 0.13671875, "grad_norm": 1.4867818355560303, "learning_rate": 9.992417937414932e-05, "loss": 0.09548898041248322, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.1002, "step": 35, "tokens/total": 1139824, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 15891} +{"epoch": 0.140625, "grad_norm": 0.6409967541694641, "learning_rate": 9.99064004568618e-05, "loss": 0.07961225509643555, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.08287, "step": 36, "tokens/total": 1176264, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 16371} +{"epoch": 0.14453125, "grad_norm": 0.9221198558807373, "learning_rate": 9.988675280095074e-05, "loss": 0.10356143862009048, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.10911, "step": 37, "tokens/total": 1209016, "tokens/train_per_sec_per_gpu": 10.15, "tokens/trainable": 16821} +{"epoch": 0.1484375, "grad_norm": 2.7015814781188965, "learning_rate": 9.986523722403528e-05, "loss": 0.08112752437591553, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.08451, "step": 38, "tokens/total": 1243448, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 17282} +{"epoch": 0.15234375, "grad_norm": 0.5462889075279236, "learning_rate": 9.984185462146642e-05, "loss": 0.08813770860433578, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.09214, "step": 39, "tokens/total": 1279216, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 17767} +{"epoch": 0.15625, "grad_norm": 0.9077551364898682, "learning_rate": 9.98166059662897e-05, "loss": 0.09378387778997421, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.09832, "step": 40, "tokens/total": 1310072, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 18210} +{"epoch": 0.16015625, "grad_norm": 1.0047416687011719, "learning_rate": 9.978949230920472e-05, "loss": 0.07014170289039612, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.25, "memory/max_allocated (GiB)": 64.25, "ppl": 1.07266, "step": 41, "tokens/total": 1343224, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 18690} +{"epoch": 0.1640625, "grad_norm": 0.7240456342697144, "learning_rate": 9.976051477852141e-05, "loss": 0.09799335896968842, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.10296, "step": 42, "tokens/total": 1376168, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 19088} +{"epoch": 0.16796875, "grad_norm": 0.3835766315460205, "learning_rate": 9.972967458011312e-05, "loss": 0.0659402385354042, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.06816, "step": 43, "tokens/total": 1409752, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 19565} +{"epoch": 0.171875, "grad_norm": 1.0044331550598145, "learning_rate": 9.96969729973664e-05, "loss": 0.08058324456214905, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.54, "memory/max_allocated (GiB)": 64.54, "ppl": 1.08392, "step": 44, "tokens/total": 1440264, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 20020} +{"epoch": 0.17578125, "grad_norm": 2.074944019317627, "learning_rate": 9.966241139112754e-05, "loss": 0.1097932904958725, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.11605, "step": 45, "tokens/total": 1472984, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 20444} +{"epoch": 0.1796875, "grad_norm": 0.5471498370170593, "learning_rate": 9.96259911996461e-05, "loss": 0.08125104010105133, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.08464, "step": 46, "tokens/total": 1508056, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 20899} +{"epoch": 0.18359375, "grad_norm": 0.47061818838119507, "learning_rate": 9.958771393851491e-05, "loss": 0.06585986912250519, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.06808, "step": 47, "tokens/total": 1541600, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 21354} +{"epoch": 0.1875, "grad_norm": 0.6256953477859497, "learning_rate": 9.954758120060702e-05, "loss": 0.08844157308340073, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.09247, "step": 48, "tokens/total": 1571208, "tokens/train_per_sec_per_gpu": 10.19, "tokens/trainable": 21794} +{"epoch": 0.19140625, "grad_norm": 0.5885289907455444, "learning_rate": 9.950559465600948e-05, "loss": 0.08439810574054718, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.08806, "step": 49, "tokens/total": 1607248, "tokens/train_per_sec_per_gpu": 6.69, "tokens/trainable": 22268} +{"epoch": 0.1953125, "grad_norm": 1.6458981037139893, "learning_rate": 9.946175605195379e-05, "loss": 0.06443492323160172, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.06656, "step": 50, "tokens/total": 1639840, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 22747} +{"epoch": 0.19921875, "grad_norm": 0.5577486157417297, "learning_rate": 9.941606721274322e-05, "loss": 0.07988445460796356, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.31, "memory/max_allocated (GiB)": 65.31, "ppl": 1.08316, "step": 51, "tokens/total": 1672896, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 23246} +{"epoch": 0.203125, "grad_norm": 0.6619119644165039, "learning_rate": 9.936853003967685e-05, "loss": 0.07025793939828873, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.07278, "step": 52, "tokens/total": 1708568, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 23725} +{"epoch": 0.20703125, "grad_norm": 0.8265272378921509, "learning_rate": 9.93191465109705e-05, "loss": 0.08790676295757294, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.09189, "step": 53, "tokens/total": 1742696, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 24152} +{"epoch": 0.2109375, "grad_norm": 0.7645062208175659, "learning_rate": 9.926791868167438e-05, "loss": 0.07093480974435806, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.07351, "step": 54, "tokens/total": 1778144, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 24603} +{"epoch": 0.21484375, "grad_norm": 0.8038988709449768, "learning_rate": 9.921484868358753e-05, "loss": 0.06653305143117905, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.0688, "step": 55, "tokens/total": 1812824, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 25026} +{"epoch": 0.21875, "grad_norm": 0.8285092115402222, "learning_rate": 9.915993872516924e-05, "loss": 0.06508304178714752, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.06725, "step": 56, "tokens/total": 1845720, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 25495} +{"epoch": 0.22265625, "grad_norm": 1.4561554193496704, "learning_rate": 9.9103191091447e-05, "loss": 0.08358393609523773, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.08718, "step": 57, "tokens/total": 1877800, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 25968} +{"epoch": 0.2265625, "grad_norm": 0.8698760271072388, "learning_rate": 9.904460814392147e-05, "loss": 0.06765519827604294, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.07, "step": 58, "tokens/total": 1907936, "tokens/train_per_sec_per_gpu": 10.88, "tokens/trainable": 26418} +{"epoch": 0.23046875, "grad_norm": 0.4061572849750519, "learning_rate": 9.898419232046825e-05, "loss": 0.030959703028202057, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.03144, "step": 59, "tokens/total": 1939848, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 26866} +{"epoch": 0.234375, "grad_norm": 0.7237622141838074, "learning_rate": 9.892194613523633e-05, "loss": 0.06298012286424637, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.06501, "step": 60, "tokens/total": 1971560, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 27332} +{"epoch": 0.23828125, "grad_norm": 0.4559035897254944, "learning_rate": 9.885787217854357e-05, "loss": 0.041379205882549286, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.04225, "step": 61, "tokens/total": 2000352, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 27734} +{"epoch": 0.2421875, "grad_norm": 0.6591630578041077, "learning_rate": 9.879197311676887e-05, "loss": 0.03343372792005539, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.034, "step": 62, "tokens/total": 2032616, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 28217} +{"epoch": 0.24609375, "grad_norm": 0.5142337679862976, "learning_rate": 9.872425169224113e-05, "loss": 0.02992893196642399, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.03038, "step": 63, "tokens/total": 2065968, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 28696} +{"epoch": 0.25, "grad_norm": 1.3511276245117188, "learning_rate": 9.865471072312528e-05, "loss": 0.04433634877204895, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.04533, "step": 64, "tokens/total": 2096760, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 29144} +{"epoch": 0.25390625, "grad_norm": 2.4607627391815186, "learning_rate": 9.858335310330492e-05, "loss": 0.05017475411295891, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.05145, "step": 65, "tokens/total": 2124976, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 29569} +{"epoch": 0.2578125, "grad_norm": 1.7109893560409546, "learning_rate": 9.851018180226185e-05, "loss": 0.05933705344796181, "memory/device_reserved (GiB)": 66.37, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.06113, "step": 66, "tokens/total": 2153384, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 30010} +{"epoch": 0.26171875, "grad_norm": 0.7918607592582703, "learning_rate": 9.843519986495259e-05, "loss": 0.017320845276117325, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 65.69, "memory/max_allocated (GiB)": 65.69, "ppl": 1.01747, "step": 67, "tokens/total": 2186688, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 30466} +{"epoch": 0.265625, "grad_norm": 3.5381102561950684, "learning_rate": 9.835841041168162e-05, "loss": 0.027242625132203102, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.02762, "step": 68, "tokens/total": 2219192, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 30943} +{"epoch": 0.26953125, "grad_norm": 0.8843804597854614, "learning_rate": 9.82798166379715e-05, "loss": 0.0274767205119133, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.02786, "step": 69, "tokens/total": 2249048, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 31364} +{"epoch": 0.2734375, "grad_norm": 0.3677317798137665, "learning_rate": 9.819942181443002e-05, "loss": 0.006923864595592022, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00695, "step": 70, "tokens/total": 2280936, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 31782} +{"epoch": 0.27734375, "grad_norm": 0.8564284443855286, "learning_rate": 9.811722928661392e-05, "loss": 0.013134637847542763, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.01322, "step": 71, "tokens/total": 2312920, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 32257} +{"epoch": 0.28125, "grad_norm": 1.277357816696167, "learning_rate": 9.803324247488975e-05, "loss": 0.041127026081085205, "memory/device_reserved (GiB)": 66.62, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.04198, "step": 72, "tokens/total": 2345864, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 32707} +{"epoch": 0.28515625, "grad_norm": 1.1520013809204102, "learning_rate": 9.794746487429161e-05, "loss": 0.04431108757853508, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.04531, "step": 73, "tokens/total": 2379328, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 33207} +{"epoch": 0.2890625, "grad_norm": 1.2973333597183228, "learning_rate": 9.785990005437554e-05, "loss": 0.03889952227473259, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.03967, "step": 74, "tokens/total": 2409520, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 33646} +{"epoch": 0.29296875, "grad_norm": 0.7411920428276062, "learning_rate": 9.777055165907117e-05, "loss": 0.04875597730278969, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.04996, "step": 75, "tokens/total": 2442240, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 34112} +{"epoch": 0.296875, "grad_norm": 6.509432315826416, "learning_rate": 9.767942340652993e-05, "loss": 0.02761002443730831, "memory/device_reserved (GiB)": 66.74, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.02799, "step": 76, "tokens/total": 2474616, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 34592} +{"epoch": 0.30078125, "grad_norm": 1.6267874240875244, "learning_rate": 9.758651908897035e-05, "loss": 0.030791670083999634, "memory/device_reserved (GiB)": 66.84, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.03127, "step": 77, "tokens/total": 2509184, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 35038} +{"epoch": 0.3046875, "grad_norm": 22.415111541748047, "learning_rate": 9.749184257252033e-05, "loss": 0.02178378775715828, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.02202, "step": 78, "tokens/total": 2543744, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 35529} +{"epoch": 0.30859375, "grad_norm": 2.697216272354126, "learning_rate": 9.739539779705614e-05, "loss": 0.039816729724407196, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.04062, "step": 79, "tokens/total": 2577192, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 35960} +{"epoch": 0.3125, "grad_norm": 1.3221787214279175, "learning_rate": 9.729718877603861e-05, "loss": 0.03796793520450592, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.0387, "step": 80, "tokens/total": 2610528, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 36405} +{"epoch": 0.31640625, "grad_norm": 1.0528324842453003, "learning_rate": 9.719721959634592e-05, "loss": 0.018342599272727966, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01851, "step": 81, "tokens/total": 2641280, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 36852} +{"epoch": 0.3203125, "grad_norm": 2.103630542755127, "learning_rate": 9.709549441810375e-05, "loss": 0.04233074188232422, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.04324, "step": 82, "tokens/total": 2671824, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 37291} +{"epoch": 0.32421875, "grad_norm": 1.1355125904083252, "learning_rate": 9.699201747451195e-05, "loss": 0.034272532910108566, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.03487, "step": 83, "tokens/total": 2704016, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 37749} +{"epoch": 0.328125, "grad_norm": 0.5819991827011108, "learning_rate": 9.688679307166854e-05, "loss": 0.014805449172854424, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01492, "step": 84, "tokens/total": 2736504, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 38205} +{"epoch": 0.33203125, "grad_norm": 0.7586485147476196, "learning_rate": 9.677982558839042e-05, "loss": 0.02796734683215618, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.02836, "step": 85, "tokens/total": 2770552, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 38674} +{"epoch": 0.3359375, "grad_norm": 1.5498201847076416, "learning_rate": 9.66711194760312e-05, "loss": 0.04247776046395302, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 64.19, "memory/max_allocated (GiB)": 64.19, "ppl": 1.04339, "step": 86, "tokens/total": 2803304, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 39124} +{"epoch": 0.33984375, "grad_norm": 1.583512783050537, "learning_rate": 9.656067925829593e-05, "loss": 0.02355056069791317, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.02383, "step": 87, "tokens/total": 2833864, "tokens/train_per_sec_per_gpu": 10.81, "tokens/trainable": 39600} +{"epoch": 0.34375, "grad_norm": 1.3539608716964722, "learning_rate": 9.644850953105288e-05, "loss": 0.045951131731271744, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.04702, "step": 88, "tokens/total": 2864544, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 40068} +{"epoch": 0.34765625, "grad_norm": 0.8181082010269165, "learning_rate": 9.633461496214225e-05, "loss": 0.009883432649075985, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.00993, "step": 89, "tokens/total": 2891600, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 40452} +{"epoch": 0.3515625, "grad_norm": 1.3910359144210815, "learning_rate": 9.621900029118195e-05, "loss": 0.026719827204942703, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.02708, "step": 90, "tokens/total": 2925328, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 40925} +{"epoch": 0.35546875, "grad_norm": 1.2895621061325073, "learning_rate": 9.610167032937036e-05, "loss": 0.02018004283308983, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.02039, "step": 91, "tokens/total": 2957904, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 41374} +{"epoch": 0.359375, "grad_norm": 1.4247550964355469, "learning_rate": 9.598262995928611e-05, "loss": 0.016791993752121925, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.01693, "step": 92, "tokens/total": 2989496, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 41807} +{"epoch": 0.36328125, "grad_norm": 1.12920343875885, "learning_rate": 9.586188413468492e-05, "loss": 0.019220663234591484, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.01941, "step": 93, "tokens/total": 3021704, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 42236} +{"epoch": 0.3671875, "grad_norm": 1.47605562210083, "learning_rate": 9.57394378802934e-05, "loss": 0.052437156438827515, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.63, "memory/max_allocated (GiB)": 65.63, "ppl": 1.05384, "step": 94, "tokens/total": 3055864, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 42712} +{"epoch": 0.37109375, "grad_norm": 2.3529248237609863, "learning_rate": 9.56152962916e-05, "loss": 0.02205880917608738, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.77, "memory/max_allocated (GiB)": 64.77, "ppl": 1.0223, "step": 95, "tokens/total": 3089592, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 43213} +{"epoch": 0.375, "grad_norm": 0.9685836434364319, "learning_rate": 9.548946453464296e-05, "loss": 0.027760041877627373, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.02815, "step": 96, "tokens/total": 3124216, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 43702} +{"epoch": 0.37890625, "grad_norm": 2.1465823650360107, "learning_rate": 9.53619478457953e-05, "loss": 0.02974855899810791, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.0302, "step": 97, "tokens/total": 3157360, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 44158} +{"epoch": 0.3828125, "grad_norm": 5.5856404304504395, "learning_rate": 9.523275153154695e-05, "loss": 0.026176247745752335, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.55, "memory/max_allocated (GiB)": 64.55, "ppl": 1.02652, "step": 98, "tokens/total": 3188416, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 44615} +{"epoch": 0.38671875, "grad_norm": 2.459883213043213, "learning_rate": 9.51018809682839e-05, "loss": 0.011881689541041851, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.11, "memory/max_allocated (GiB)": 63.11, "ppl": 1.01195, "step": 99, "tokens/total": 3214280, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 45042} +{"epoch": 0.390625, "grad_norm": 1.1651434898376465, "learning_rate": 9.49693416020645e-05, "loss": 0.018559589982032776, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.01873, "step": 100, "tokens/total": 3248168, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 45508} +{"epoch": 0.39453125, "grad_norm": 1.4064868688583374, "learning_rate": 9.483513894839276e-05, "loss": 0.03029109165072441, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.03075, "step": 101, "tokens/total": 3280240, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 45989} +{"epoch": 0.3984375, "grad_norm": 1.016870141029358, "learning_rate": 9.469927859198888e-05, "loss": 0.023595916107296944, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.02388, "step": 102, "tokens/total": 3311424, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 46425} +{"epoch": 0.40234375, "grad_norm": 0.402317076921463, "learning_rate": 9.456176618655689e-05, "loss": 0.0019403960322961211, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00194, "step": 103, "tokens/total": 3345632, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 46876} +{"epoch": 0.40625, "grad_norm": 1.2395771741867065, "learning_rate": 9.442260745454927e-05, "loss": 0.009496588259935379, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.46, "memory/max_allocated (GiB)": 65.46, "ppl": 1.00954, "step": 104, "tokens/total": 3379208, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 47378} +{"epoch": 0.41015625, "grad_norm": 1.6285722255706787, "learning_rate": 9.428180818692884e-05, "loss": 0.016758766025304794, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.0169, "step": 105, "tokens/total": 3409760, "tokens/train_per_sec_per_gpu": 10.59, "tokens/trainable": 47852} +{"epoch": 0.4140625, "grad_norm": 1.6795839071273804, "learning_rate": 9.413937424292791e-05, "loss": 0.02724863961338997, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.02762, "step": 106, "tokens/total": 3441168, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 48325} +{"epoch": 0.41796875, "grad_norm": 1.5863374471664429, "learning_rate": 9.399531154980424e-05, "loss": 0.014184612780809402, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.01429, "step": 107, "tokens/total": 3474904, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 48818} +{"epoch": 0.421875, "grad_norm": 8.595233917236328, "learning_rate": 9.384962610259455e-05, "loss": 0.007445233874022961, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.54, "memory/max_allocated (GiB)": 65.54, "ppl": 1.00747, "step": 108, "tokens/total": 3507856, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 49256} +{"epoch": 0.42578125, "grad_norm": 0.7784904837608337, "learning_rate": 9.370232396386494e-05, "loss": 0.028034742921590805, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.02843, "step": 109, "tokens/total": 3542296, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 49701} +{"epoch": 0.4296875, "grad_norm": 3.2436234951019287, "learning_rate": 9.355341126345868e-05, "loss": 0.05168883129954338, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.05305, "step": 110, "tokens/total": 3577224, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 50192} +{"epoch": 0.43359375, "grad_norm": 0.5394429564476013, "learning_rate": 9.340289419824107e-05, "loss": 0.010503975674510002, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.01056, "step": 111, "tokens/total": 3608136, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 50608} +{"epoch": 0.4375, "grad_norm": 1.1756888628005981, "learning_rate": 9.325077903184159e-05, "loss": 0.02183087170124054, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.02207, "step": 112, "tokens/total": 3637736, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 51032} +{"epoch": 0.44140625, "grad_norm": 1.0380253791809082, "learning_rate": 9.30970720943932e-05, "loss": 0.0038046417757868767, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00381, "step": 113, "tokens/total": 3671112, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 51537} +{"epoch": 0.4453125, "grad_norm": 2.846378803253174, "learning_rate": 9.2941779782269e-05, "loss": 0.05107184499502182, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.87, "memory/max_allocated (GiB)": 63.87, "ppl": 1.0524, "step": 114, "tokens/total": 3701080, "tokens/train_per_sec_per_gpu": 10.07, "tokens/trainable": 52004} +{"epoch": 0.44921875, "grad_norm": 2.3435041904449463, "learning_rate": 9.278490855781596e-05, "loss": 0.017886217683553696, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.01805, "step": 115, "tokens/total": 3734416, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 52487} +{"epoch": 0.453125, "grad_norm": 1.2350008487701416, "learning_rate": 9.262646494908604e-05, "loss": 0.028789594769477844, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.02921, "step": 116, "tokens/total": 3770664, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 52965} +{"epoch": 0.45703125, "grad_norm": 0.721799910068512, "learning_rate": 9.246645554956457e-05, "loss": 0.023434441536664963, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.02371, "step": 117, "tokens/total": 3803832, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 53450} +{"epoch": 0.4609375, "grad_norm": 1.4129279851913452, "learning_rate": 9.230488701789578e-05, "loss": 0.010564781725406647, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01062, "step": 118, "tokens/total": 3837248, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 53927} +{"epoch": 0.46484375, "grad_norm": 0.7282304167747498, "learning_rate": 9.214176607760577e-05, "loss": 0.011499300599098206, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.01157, "step": 119, "tokens/total": 3871512, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 54395} +{"epoch": 0.46875, "grad_norm": 0.6591880917549133, "learning_rate": 9.197709951682268e-05, "loss": 0.018765881657600403, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.01894, "step": 120, "tokens/total": 3907056, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 54871} +{"epoch": 0.47265625, "grad_norm": 1.1185778379440308, "learning_rate": 9.181089418799428e-05, "loss": 0.021032849326729774, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.02126, "step": 121, "tokens/total": 3938872, "tokens/train_per_sec_per_gpu": 10.27, "tokens/trainable": 55356} +{"epoch": 0.4765625, "grad_norm": 0.3383438289165497, "learning_rate": 9.164315700760271e-05, "loss": 0.003726528026163578, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.00373, "step": 122, "tokens/total": 3971928, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 55836} +{"epoch": 0.48046875, "grad_norm": 10.71234130859375, "learning_rate": 9.147389495587671e-05, "loss": 0.013200334273278713, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.87, "memory/max_allocated (GiB)": 63.87, "ppl": 1.01329, "step": 123, "tokens/total": 4000432, "tokens/train_per_sec_per_gpu": 10.09, "tokens/trainable": 56275} +{"epoch": 0.484375, "grad_norm": 0.9178401827812195, "learning_rate": 9.130311507650116e-05, "loss": 0.0020134705118834972, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00202, "step": 124, "tokens/total": 4032984, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 56745} +{"epoch": 0.48828125, "grad_norm": 1.5906853675842285, "learning_rate": 9.113082447632394e-05, "loss": 0.03583294525742531, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.29, "memory/max_allocated (GiB)": 65.29, "ppl": 1.03648, "step": 125, "tokens/total": 4067024, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 57171} +{"epoch": 0.4921875, "grad_norm": 0.9905876517295837, "learning_rate": 9.09570303250602e-05, "loss": 0.009167100302875042, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00921, "step": 126, "tokens/total": 4098808, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 57642} +{"epoch": 0.49609375, "grad_norm": 0.9293321967124939, "learning_rate": 9.078173985499394e-05, "loss": 0.0016866277437657118, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00169, "step": 127, "tokens/total": 4131768, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 58134} +{"epoch": 0.5, "grad_norm": 21.202341079711914, "learning_rate": 9.060496036067713e-05, "loss": 0.007621073629707098, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00765, "step": 128, "tokens/total": 4164176, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 58564} +{"epoch": 0.50390625, "grad_norm": 1.2851237058639526, "learning_rate": 9.042669919862615e-05, "loss": 0.007170678582042456, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.69, "memory/max_allocated (GiB)": 64.69, "ppl": 1.0072, "step": 129, "tokens/total": 4195608, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 59038} +{"epoch": 0.5078125, "grad_norm": 6.877071380615234, "learning_rate": 9.024696378701557e-05, "loss": 0.001891867257654667, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00189, "step": 130, "tokens/total": 4227600, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 59495} +{"epoch": 0.51171875, "grad_norm": 1.259360432624817, "learning_rate": 9.006576160536948e-05, "loss": 0.0205297339707613, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.02074, "step": 131, "tokens/total": 4257936, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 59911} +{"epoch": 0.515625, "grad_norm": 1.293566107749939, "learning_rate": 8.988310019425035e-05, "loss": 0.00453556701540947, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00455, "step": 132, "tokens/total": 4288360, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 60371} +{"epoch": 0.51953125, "grad_norm": 1.7815507650375366, "learning_rate": 8.969898715494506e-05, "loss": 0.031245006248354912, "memory/device_reserved (GiB)": 65.79, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.03174, "step": 133, "tokens/total": 4321224, "tokens/train_per_sec_per_gpu": 10.04, "tokens/trainable": 60827} +{"epoch": 0.5234375, "grad_norm": 0.8443915843963623, "learning_rate": 8.951343014914869e-05, "loss": 0.011514952406287193, "memory/device_reserved (GiB)": 65.93, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.01158, "step": 134, "tokens/total": 4354544, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 61320} +{"epoch": 0.52734375, "grad_norm": 1.8312216997146606, "learning_rate": 8.932643689864568e-05, "loss": 0.01712869293987751, "memory/device_reserved (GiB)": 65.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.01728, "step": 135, "tokens/total": 4386120, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 61775} +{"epoch": 0.53125, "grad_norm": 0.3754090666770935, "learning_rate": 8.913801518498845e-05, "loss": 0.0029303887858986855, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00293, "step": 136, "tokens/total": 4420072, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 62248} +{"epoch": 0.53515625, "grad_norm": 0.43289026618003845, "learning_rate": 8.894817284917364e-05, "loss": 0.003482871688902378, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00349, "step": 137, "tokens/total": 4456800, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 62730} +{"epoch": 0.5390625, "grad_norm": 0.9994592070579529, "learning_rate": 8.875691779131569e-05, "loss": 0.020444713532924652, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.02066, "step": 138, "tokens/total": 4493688, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 63166} +{"epoch": 0.54296875, "grad_norm": 3.516019582748413, "learning_rate": 8.856425797031829e-05, "loss": 0.042043287307024, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.04294, "step": 139, "tokens/total": 4524896, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 63631} +{"epoch": 0.546875, "grad_norm": 0.9377413392066956, "learning_rate": 8.837020140354295e-05, "loss": 0.03427302837371826, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.03487, "step": 140, "tokens/total": 4558704, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 64080} +{"epoch": 0.55078125, "grad_norm": 0.6889838576316833, "learning_rate": 8.817475616647554e-05, "loss": 0.014078070409595966, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.01418, "step": 141, "tokens/total": 4590824, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 64537} +{"epoch": 0.5546875, "grad_norm": 0.7547942399978638, "learning_rate": 8.797793039239017e-05, "loss": 0.020853225141763687, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.02107, "step": 142, "tokens/total": 4622328, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 65011} +{"epoch": 0.55859375, "grad_norm": 1.0266203880310059, "learning_rate": 8.777973227201069e-05, "loss": 0.008182761259377003, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.7, "memory/max_allocated (GiB)": 64.7, "ppl": 1.00822, "step": 143, "tokens/total": 4654840, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 65511} +{"epoch": 0.5625, "grad_norm": 0.5139380693435669, "learning_rate": 8.758017005316988e-05, "loss": 0.00790315866470337, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00793, "step": 144, "tokens/total": 4688200, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 65996} +{"epoch": 0.56640625, "grad_norm": 1.3175464868545532, "learning_rate": 8.737925204046629e-05, "loss": 0.011800430715084076, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.01187, "step": 145, "tokens/total": 4719400, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 66423} +{"epoch": 0.5703125, "grad_norm": 0.5768738389015198, "learning_rate": 8.717698659491851e-05, "loss": 0.008091442286968231, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00812, "step": 146, "tokens/total": 4753928, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 66905} +{"epoch": 0.57421875, "grad_norm": 0.6783992052078247, "learning_rate": 8.697338213361735e-05, "loss": 0.0028052006382495165, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00281, "step": 147, "tokens/total": 4788456, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 67363} +{"epoch": 0.578125, "grad_norm": 0.27097225189208984, "learning_rate": 8.676844712937552e-05, "loss": 0.006871978752315044, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.62, "memory/max_allocated (GiB)": 64.62, "ppl": 1.0069, "step": 148, "tokens/total": 4820096, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 67829} +{"epoch": 0.58203125, "grad_norm": 3.299593448638916, "learning_rate": 8.656219011037509e-05, "loss": 0.003326334757730365, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00333, "step": 149, "tokens/total": 4850208, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 68286} +{"epoch": 0.5859375, "grad_norm": 1.1381242275238037, "learning_rate": 8.63546196598125e-05, "loss": 0.012367094866931438, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01244, "step": 150, "tokens/total": 4884720, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 68750} +{"epoch": 0.58984375, "grad_norm": 0.904266893863678, "learning_rate": 8.614574441554145e-05, "loss": 0.011679286137223244, "memory/device_reserved (GiB)": 66.83, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.01175, "step": 151, "tokens/total": 4922000, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 69231} +{"epoch": 0.59375, "grad_norm": 1.217503309249878, "learning_rate": 8.593557306971349e-05, "loss": 0.01694379933178425, "memory/device_reserved (GiB)": 66.83, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.01709, "step": 152, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 69711} +{"epoch": 0.59765625, "grad_norm": 0.15243332087993622, "learning_rate": 8.572411436841618e-05, "loss": 0.0007724524475634098, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00077, "step": 153, "tokens/total": 4992880, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 70178} +{"epoch": 0.6015625, "grad_norm": 0.8899665474891663, "learning_rate": 8.551137711130922e-05, "loss": 0.013538237661123276, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01363, "step": 154, "tokens/total": 5025136, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 70670} +{"epoch": 0.60546875, "grad_norm": 0.7498807311058044, "learning_rate": 8.529737015125824e-05, "loss": 0.014306997880339622, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01441, "step": 155, "tokens/total": 5058912, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 71111} +{"epoch": 0.609375, "grad_norm": 0.5268966555595398, "learning_rate": 8.508210239396639e-05, "loss": 0.005755068268626928, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.00577, "step": 156, "tokens/total": 5091432, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 71623} +{"epoch": 0.61328125, "grad_norm": 0.6768266558647156, "learning_rate": 8.486558279760375e-05, "loss": 0.004716098308563232, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00473, "step": 157, "tokens/total": 5125920, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 72100} +{"epoch": 0.6171875, "grad_norm": 0.37949109077453613, "learning_rate": 8.464782037243449e-05, "loss": 0.006338577251881361, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00636, "step": 158, "tokens/total": 5159664, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 72598} +{"epoch": 0.62109375, "grad_norm": 2.1656947135925293, "learning_rate": 8.442882418044202e-05, "loss": 0.011411416344344616, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.01148, "step": 159, "tokens/total": 5195584, "tokens/train_per_sec_per_gpu": 6.81, "tokens/trainable": 73071} +{"epoch": 0.625, "grad_norm": 4.967442035675049, "learning_rate": 8.420860333495179e-05, "loss": 0.009260348975658417, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.0093, "step": 160, "tokens/total": 5229664, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 73560} +{"epoch": 0.62890625, "grad_norm": 0.5787362456321716, "learning_rate": 8.398716700025208e-05, "loss": 0.003804129082709551, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00381, "step": 161, "tokens/total": 5264008, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 74083} +{"epoch": 0.6328125, "grad_norm": 0.7844538688659668, "learning_rate": 8.376452439121266e-05, "loss": 0.020713426172733307, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.02093, "step": 162, "tokens/total": 5294720, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 74530} +{"epoch": 0.63671875, "grad_norm": 0.7800741791725159, "learning_rate": 8.354068477290124e-05, "loss": 0.011643832549452782, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.01171, "step": 163, "tokens/total": 5325376, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 74999} +{"epoch": 0.640625, "grad_norm": 0.9846333265304565, "learning_rate": 8.331565746019807e-05, "loss": 0.015045375563204288, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.01516, "step": 164, "tokens/total": 5360240, "tokens/train_per_sec_per_gpu": 7.02, "tokens/trainable": 75450} +{"epoch": 0.64453125, "grad_norm": 0.6700380444526672, "learning_rate": 8.308945181740812e-05, "loss": 0.004570442251861095, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00458, "step": 165, "tokens/total": 5392376, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 75921} +{"epoch": 0.6484375, "grad_norm": 1.5390267372131348, "learning_rate": 8.286207725787153e-05, "loss": 0.014085885137319565, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.01419, "step": 166, "tokens/total": 5427456, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 76425} +{"epoch": 0.65234375, "grad_norm": 0.6424888372421265, "learning_rate": 8.263354324357182e-05, "loss": 0.014942290261387825, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.01505, "step": 167, "tokens/total": 5457688, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 76863} +{"epoch": 0.65625, "grad_norm": 0.8840211629867554, "learning_rate": 8.240385928474219e-05, "loss": 0.013696830719709396, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01379, "step": 168, "tokens/total": 5488600, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 77298} +{"epoch": 0.66015625, "grad_norm": 0.5733786821365356, "learning_rate": 8.217303493946967e-05, "loss": 0.009710732847452164, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00976, "step": 169, "tokens/total": 5521856, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 77735} +{"epoch": 0.6640625, "grad_norm": 0.23610819876194, "learning_rate": 8.194107981329746e-05, "loss": 0.002865488873794675, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00287, "step": 170, "tokens/total": 5553272, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 78179} +{"epoch": 0.66796875, "grad_norm": 0.8329737782478333, "learning_rate": 8.170800355882518e-05, "loss": 0.006768189836293459, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.57, "memory/max_allocated (GiB)": 65.57, "ppl": 1.00679, "step": 171, "tokens/total": 5587776, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 78660} +{"epoch": 0.671875, "grad_norm": 0.692070484161377, "learning_rate": 8.147381587530713e-05, "loss": 0.006605828646570444, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00663, "step": 172, "tokens/total": 5620864, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 79110} +{"epoch": 0.67578125, "grad_norm": 2.9117817878723145, "learning_rate": 8.123852650824877e-05, "loss": 0.015457206405699253, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.01558, "step": 173, "tokens/total": 5653656, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 79576} +{"epoch": 0.6796875, "grad_norm": 0.21615418791770935, "learning_rate": 8.100214524900103e-05, "loss": 0.0024237162433564663, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00243, "step": 174, "tokens/total": 5682312, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 80001} +{"epoch": 0.68359375, "grad_norm": 0.8334981203079224, "learning_rate": 8.076468193435301e-05, "loss": 0.007844610139727592, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00788, "step": 175, "tokens/total": 5713520, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 80452} +{"epoch": 0.6875, "grad_norm": 6.801231861114502, "learning_rate": 8.052614644612253e-05, "loss": 0.011320993304252625, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.01139, "step": 176, "tokens/total": 5745296, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 80886} +{"epoch": 0.69140625, "grad_norm": 1.4458545446395874, "learning_rate": 8.028654871074489e-05, "loss": 0.00917469896376133, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00922, "step": 177, "tokens/total": 5774336, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 81325} +{"epoch": 0.6953125, "grad_norm": 0.3217310905456543, "learning_rate": 8.004589869885986e-05, "loss": 0.0011722161434590816, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00117, "step": 178, "tokens/total": 5805032, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 81732} +{"epoch": 0.69921875, "grad_norm": 0.04811696335673332, "learning_rate": 7.980420642489674e-05, "loss": 0.00020472623873502016, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.0002, "step": 179, "tokens/total": 5832616, "tokens/train_per_sec_per_gpu": 11.15, "tokens/trainable": 82171} +{"epoch": 0.703125, "grad_norm": 1.0301192998886108, "learning_rate": 7.95614819466576e-05, "loss": 0.018128203228116035, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01829, "step": 180, "tokens/total": 5865568, "tokens/train_per_sec_per_gpu": 10.49, "tokens/trainable": 82670} +{"epoch": 0.70703125, "grad_norm": 0.6469829678535461, "learning_rate": 7.931773536489872e-05, "loss": 0.00764002650976181, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00767, "step": 181, "tokens/total": 5894256, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 83084} +{"epoch": 0.7109375, "grad_norm": 0.39863279461860657, "learning_rate": 7.907297682291035e-05, "loss": 0.0012292676838114858, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00123, "step": 182, "tokens/total": 5924304, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 83500} +{"epoch": 0.71484375, "grad_norm": 0.5780090093612671, "learning_rate": 7.882721650609442e-05, "loss": 0.004889205098152161, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.0049, "step": 183, "tokens/total": 5955128, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 83896} +{"epoch": 0.71875, "grad_norm": 1.6885523796081543, "learning_rate": 7.85804646415409e-05, "loss": 0.051915742456912994, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.86, "memory/max_allocated (GiB)": 64.86, "ppl": 1.05329, "step": 184, "tokens/total": 5988056, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 84399} +{"epoch": 0.72265625, "grad_norm": 0.3248288333415985, "learning_rate": 7.833273149760207e-05, "loss": 0.0042898403480648994, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.0043, "step": 185, "tokens/total": 6019544, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 84873} +{"epoch": 0.7265625, "grad_norm": 2.445359468460083, "learning_rate": 7.808402738346527e-05, "loss": 0.027457276359200478, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.02784, "step": 186, "tokens/total": 6051616, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 85296} +{"epoch": 0.73046875, "grad_norm": 0.665398895740509, "learning_rate": 7.783436264872382e-05, "loss": 0.008853345178067684, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.54, "memory/max_allocated (GiB)": 64.54, "ppl": 1.00889, "step": 187, "tokens/total": 6084304, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 85741} +{"epoch": 0.734375, "grad_norm": 0.6446435451507568, "learning_rate": 7.758374768294647e-05, "loss": 0.005346850026398897, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00536, "step": 188, "tokens/total": 6117896, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 86176} +{"epoch": 0.73828125, "grad_norm": 0.3597274124622345, "learning_rate": 7.733219291524489e-05, "loss": 0.005265028681606054, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00528, "step": 189, "tokens/total": 6152296, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 86649} +{"epoch": 0.7421875, "grad_norm": 0.29664063453674316, "learning_rate": 7.707970881383977e-05, "loss": 0.003340602619573474, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00335, "step": 190, "tokens/total": 6186056, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 87084} +{"epoch": 0.74609375, "grad_norm": 5.040450096130371, "learning_rate": 7.682630588562518e-05, "loss": 0.026267290115356445, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.02662, "step": 191, "tokens/total": 6219160, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 87499} +{"epoch": 0.75, "grad_norm": 0.2170778065919876, "learning_rate": 7.657199467573129e-05, "loss": 0.002919618971645832, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00292, "step": 192, "tokens/total": 6254840, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 87987} +{"epoch": 0.75390625, "grad_norm": 0.2809968888759613, "learning_rate": 7.631678576708561e-05, "loss": 0.003314524656161666, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00332, "step": 193, "tokens/total": 6289656, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 88470} +{"epoch": 0.7578125, "grad_norm": 0.4537161588668823, "learning_rate": 7.606068977997255e-05, "loss": 0.008997563272714615, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00904, "step": 194, "tokens/total": 6321776, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 88923} +{"epoch": 0.76171875, "grad_norm": 0.6309017539024353, "learning_rate": 7.580371737159148e-05, "loss": 0.006356152705848217, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00638, "step": 195, "tokens/total": 6352824, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 89338} +{"epoch": 0.765625, "grad_norm": 3.246309995651245, "learning_rate": 7.554587923561324e-05, "loss": 0.011894315481185913, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01197, "step": 196, "tokens/total": 6388496, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 89812} +{"epoch": 0.76953125, "grad_norm": 0.5897355079650879, "learning_rate": 7.528718610173511e-05, "loss": 0.004336802288889885, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00435, "step": 197, "tokens/total": 6420600, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 90252} +{"epoch": 0.7734375, "grad_norm": 0.5783535242080688, "learning_rate": 7.502764873523431e-05, "loss": 0.003972399979829788, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00398, "step": 198, "tokens/total": 6454616, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 90718} +{"epoch": 0.77734375, "grad_norm": 0.3054855465888977, "learning_rate": 7.476727793652011e-05, "loss": 0.0010512361768633127, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00105, "step": 199, "tokens/total": 6485680, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 91194} +{"epoch": 0.78125, "grad_norm": 0.3596842586994171, "learning_rate": 7.450608454068415e-05, "loss": 0.0025289885234087706, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00253, "step": 200, "tokens/total": 6518512, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 91649} +{"epoch": 0.78515625, "grad_norm": 3.7184815406799316, "learning_rate": 7.424407941704987e-05, "loss": 0.031991444528102875, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.03251, "step": 201, "tokens/total": 6548728, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 92078} +{"epoch": 0.7890625, "grad_norm": 0.7247127890586853, "learning_rate": 7.398127346871986e-05, "loss": 0.0024997720029205084, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.0025, "step": 202, "tokens/total": 6582192, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 92536} +{"epoch": 0.79296875, "grad_norm": 0.45109081268310547, "learning_rate": 7.371767763212238e-05, "loss": 0.009311533533036709, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.00936, "step": 203, "tokens/total": 6617392, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 93032} +{"epoch": 0.796875, "grad_norm": 0.12170525640249252, "learning_rate": 7.345330287655617e-05, "loss": 0.00025340961292386055, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00025, "step": 204, "tokens/total": 6648904, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 93504} +{"epoch": 0.80078125, "grad_norm": 1.3568565845489502, "learning_rate": 7.31881602037339e-05, "loss": 0.010106543079018593, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.01016, "step": 205, "tokens/total": 6679832, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 93925} +{"epoch": 0.8046875, "grad_norm": 0.2334517240524292, "learning_rate": 7.29222606473245e-05, "loss": 0.0021745553240180016, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00218, "step": 206, "tokens/total": 6710784, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 94365} +{"epoch": 0.80859375, "grad_norm": 0.2963505983352661, "learning_rate": 7.265561527249383e-05, "loss": 0.0017293061828240752, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00173, "step": 207, "tokens/total": 6744848, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 94845} +{"epoch": 0.8125, "grad_norm": 0.5881158113479614, "learning_rate": 7.238823517544436e-05, "loss": 0.005488448776304722, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.0055, "step": 208, "tokens/total": 6777600, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 95285} +{"epoch": 0.81640625, "grad_norm": 0.316368967294693, "learning_rate": 7.212013148295333e-05, "loss": 0.002648372668772936, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00265, "step": 209, "tokens/total": 6811024, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 95758} +{"epoch": 0.8203125, "grad_norm": 0.9866212606430054, "learning_rate": 7.185131535190975e-05, "loss": 0.005403297021985054, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.15, "memory/max_allocated (GiB)": 66.15, "ppl": 1.00542, "step": 210, "tokens/total": 6845376, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 96192} +{"epoch": 0.82421875, "grad_norm": 0.32367217540740967, "learning_rate": 7.158179796885005e-05, "loss": 0.0032714293338358402, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00328, "step": 211, "tokens/total": 6880776, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 96641} +{"epoch": 0.828125, "grad_norm": 0.5403663516044617, "learning_rate": 7.131159054949273e-05, "loss": 0.01425154134631157, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.01435, "step": 212, "tokens/total": 6916160, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 97160} +{"epoch": 0.83203125, "grad_norm": 0.3724394738674164, "learning_rate": 7.104070433827139e-05, "loss": 0.0035155529621988535, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.54, "memory/max_allocated (GiB)": 63.54, "ppl": 1.00352, "step": 213, "tokens/total": 6945320, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 97611} +{"epoch": 0.8359375, "grad_norm": 1.5319461822509766, "learning_rate": 7.076915060786705e-05, "loss": 0.008988060057163239, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00903, "step": 214, "tokens/total": 6977856, "tokens/train_per_sec_per_gpu": 10.01, "tokens/trainable": 98111} +{"epoch": 0.83984375, "grad_norm": 0.4524657726287842, "learning_rate": 7.049694065873882e-05, "loss": 0.0021313971374183893, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.00213, "step": 215, "tokens/total": 7011096, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 98575} +{"epoch": 0.84375, "grad_norm": 1.1272621154785156, "learning_rate": 7.022408581865382e-05, "loss": 0.004853352904319763, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.93, "memory/max_allocated (GiB)": 63.93, "ppl": 1.00487, "step": 216, "tokens/total": 7041408, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 98991} +{"epoch": 0.84765625, "grad_norm": 0.49280741810798645, "learning_rate": 6.99505974422157e-05, "loss": 0.004750218242406845, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.6, "memory/max_allocated (GiB)": 63.6, "ppl": 1.00476, "step": 217, "tokens/total": 7071560, "tokens/train_per_sec_per_gpu": 11.54, "tokens/trainable": 99454} +{"epoch": 0.8515625, "grad_norm": 0.2942127585411072, "learning_rate": 6.967648691039213e-05, "loss": 0.005255488213151693, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00527, "step": 218, "tokens/total": 7104696, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 99925} +{"epoch": 0.85546875, "grad_norm": 0.037924088537693024, "learning_rate": 6.940176563004123e-05, "loss": 0.00022039802570361644, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00022, "step": 219, "tokens/total": 7138504, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 100371} +{"epoch": 0.859375, "grad_norm": 0.5576125979423523, "learning_rate": 6.912644503343682e-05, "loss": 0.004764372482895851, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00478, "step": 220, "tokens/total": 7170992, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 100855} +{"epoch": 0.86328125, "grad_norm": 1.0339845418930054, "learning_rate": 6.885053657779273e-05, "loss": 0.0037226355634629726, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00373, "step": 221, "tokens/total": 7201176, "tokens/train_per_sec_per_gpu": 6.89, "tokens/trainable": 101253} +{"epoch": 0.8671875, "grad_norm": 0.1095302551984787, "learning_rate": 6.857405174478604e-05, "loss": 0.0009056971175596118, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00091, "step": 222, "tokens/total": 7233528, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 101706} +{"epoch": 0.87109375, "grad_norm": 1.4722586870193481, "learning_rate": 6.82970020400792e-05, "loss": 0.007482354994863272, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00751, "step": 223, "tokens/total": 7267672, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 102158} +{"epoch": 0.875, "grad_norm": 0.11905133724212646, "learning_rate": 6.801939899284132e-05, "loss": 0.0006959728780202568, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 62.86, "memory/max_allocated (GiB)": 62.86, "ppl": 1.0007, "step": 224, "tokens/total": 7295592, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 102580} +{"epoch": 0.87890625, "grad_norm": 1.6837291717529297, "learning_rate": 6.774125415526827e-05, "loss": 0.05867382511496544, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.06043, "step": 225, "tokens/total": 7325816, "tokens/train_per_sec_per_gpu": 10.07, "tokens/trainable": 103063} +{"epoch": 0.8828125, "grad_norm": 1.5482749938964844, "learning_rate": 6.746257910210214e-05, "loss": 0.014120984822511673, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.01422, "step": 226, "tokens/total": 7360888, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 103510} +{"epoch": 0.88671875, "grad_norm": 0.12654262781143188, "learning_rate": 6.718338543014937e-05, "loss": 0.0014651261735707521, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00147, "step": 227, "tokens/total": 7390416, "tokens/train_per_sec_per_gpu": 10.29, "tokens/trainable": 103980} +{"epoch": 0.890625, "grad_norm": 1.7972536087036133, "learning_rate": 6.69036847577983e-05, "loss": 0.010346658527851105, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.0104, "step": 228, "tokens/total": 7420480, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 104413} +{"epoch": 0.89453125, "grad_norm": 0.42703115940093994, "learning_rate": 6.662348872453553e-05, "loss": 0.004234187304973602, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00424, "step": 229, "tokens/total": 7448992, "tokens/train_per_sec_per_gpu": 10.2, "tokens/trainable": 104845} +{"epoch": 0.8984375, "grad_norm": 0.5800392031669617, "learning_rate": 6.63428089904618e-05, "loss": 0.01274536270648241, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.01283, "step": 230, "tokens/total": 7481744, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 105289} +{"epoch": 0.90234375, "grad_norm": 1.7588562965393066, "learning_rate": 6.60616572358065e-05, "loss": 0.038308948278427124, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.03905, "step": 231, "tokens/total": 7516704, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 105722} +{"epoch": 0.90625, "grad_norm": 0.0950864627957344, "learning_rate": 6.578004516044172e-05, "loss": 0.0010925681563094258, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.19, "memory/max_allocated (GiB)": 64.19, "ppl": 1.00109, "step": 232, "tokens/total": 7548592, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 106213} +{"epoch": 0.91015625, "grad_norm": 0.3100162744522095, "learning_rate": 6.549798448339548e-05, "loss": 0.002573960227891803, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00258, "step": 233, "tokens/total": 7579288, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 106643} +{"epoch": 0.9140625, "grad_norm": 0.3814016878604889, "learning_rate": 6.521548694236384e-05, "loss": 0.008469984866678715, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00851, "step": 234, "tokens/total": 7611672, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 107136} +{"epoch": 0.91796875, "grad_norm": 0.49100589752197266, "learning_rate": 6.493256429322259e-05, "loss": 0.010028618387877941, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.01008, "step": 235, "tokens/total": 7646288, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 107588} +{"epoch": 0.921875, "grad_norm": 0.6592451333999634, "learning_rate": 6.464922830953799e-05, "loss": 0.015556924045085907, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01568, "step": 236, "tokens/total": 7680136, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 108087} +{"epoch": 0.92578125, "grad_norm": 1.2587981224060059, "learning_rate": 6.436549078207688e-05, "loss": 0.008446453139185905, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.61, "memory/max_allocated (GiB)": 64.61, "ppl": 1.00848, "step": 237, "tokens/total": 7713072, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 108533} +{"epoch": 0.9296875, "grad_norm": 0.2825981378555298, "learning_rate": 6.408136351831592e-05, "loss": 0.0027034275699406862, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00271, "step": 238, "tokens/total": 7747856, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 108965} +{"epoch": 0.93359375, "grad_norm": 0.5357387661933899, "learning_rate": 6.379685834195036e-05, "loss": 0.008225695230066776, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00826, "step": 239, "tokens/total": 7780136, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 109416} +{"epoch": 0.9375, "grad_norm": 0.38795945048332214, "learning_rate": 6.351198709240186e-05, "loss": 0.0050722770392894745, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.78, "memory/max_allocated (GiB)": 65.78, "ppl": 1.00509, "step": 240, "tokens/total": 7816344, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 109864} +{"epoch": 0.94140625, "grad_norm": 1.2240489721298218, "learning_rate": 6.32267616243259e-05, "loss": 0.005259664263576269, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00527, "step": 241, "tokens/total": 7850408, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 110345} +{"epoch": 0.9453125, "grad_norm": 0.04203387349843979, "learning_rate": 6.294119380711849e-05, "loss": 0.0007007535314187407, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.0007, "step": 242, "tokens/total": 7881528, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 110794} +{"epoch": 0.94921875, "grad_norm": 0.1325565129518509, "learning_rate": 6.265529552442209e-05, "loss": 0.0011751661077141762, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00118, "step": 243, "tokens/total": 7915712, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 111250} +{"epoch": 0.953125, "grad_norm": 0.2977339029312134, "learning_rate": 6.236907867363127e-05, "loss": 0.005344895180314779, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.61, "memory/max_allocated (GiB)": 64.61, "ppl": 1.00536, "step": 244, "tokens/total": 7947408, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 111693} +{"epoch": 0.95703125, "grad_norm": 0.41230249404907227, "learning_rate": 6.208255516539749e-05, "loss": 0.010361938737332821, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.26, "memory/max_allocated (GiB)": 65.26, "ppl": 1.01042, "step": 245, "tokens/total": 7979600, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 112158} +{"epoch": 0.9609375, "grad_norm": 0.3992796540260315, "learning_rate": 6.179573692313344e-05, "loss": 0.0045393165200948715, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00455, "step": 246, "tokens/total": 8010848, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 112601} +{"epoch": 0.96484375, "grad_norm": 0.6036164164543152, "learning_rate": 6.150863588251694e-05, "loss": 0.0023798132315278053, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.00238, "step": 247, "tokens/total": 8040632, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 113016} +{"epoch": 0.96875, "grad_norm": 0.19013604521751404, "learning_rate": 6.122126399099419e-05, "loss": 0.0010601761750876904, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00106, "step": 248, "tokens/total": 8072464, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 113476} +{"epoch": 0.97265625, "grad_norm": 0.055494748055934906, "learning_rate": 6.0933633207282615e-05, "loss": 0.0006260098307393491, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.96, "memory/max_allocated (GiB)": 64.96, "ppl": 1.00063, "step": 249, "tokens/total": 8106568, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 113941} +{"epoch": 0.9765625, "grad_norm": 0.37465938925743103, "learning_rate": 6.064575550087316e-05, "loss": 0.0013498419430106878, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00135, "step": 250, "tokens/total": 8138496, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 114434} +{"epoch": 0.98046875, "grad_norm": 0.7683797478675842, "learning_rate": 6.0357642851532245e-05, "loss": 0.006724422797560692, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00675, "step": 251, "tokens/total": 8168080, "tokens/train_per_sec_per_gpu": 10.39, "tokens/trainable": 114898} +{"epoch": 0.984375, "grad_norm": 0.009514981880784035, "learning_rate": 6.0069307248803294e-05, "loss": 8.032267942326143e-05, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00008, "step": 252, "tokens/total": 8200888, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 115331} +{"epoch": 0.98828125, "grad_norm": 0.03063627891242504, "learning_rate": 5.9780760691507635e-05, "loss": 0.00021307730639819056, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00021, "step": 253, "tokens/total": 8230704, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 115732} +{"epoch": 0.9921875, "grad_norm": 0.513735830783844, "learning_rate": 5.9492015187245334e-05, "loss": 0.0028999578207731247, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.0029, "step": 254, "tokens/total": 8266560, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 116221} +{"epoch": 0.99609375, "grad_norm": 0.2134011834859848, "learning_rate": 5.920308275189541e-05, "loss": 0.0009439607965759933, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.19, "memory/max_allocated (GiB)": 65.19, "ppl": 1.00094, "step": 255, "tokens/total": 8300184, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 116684} +{"epoch": 1.0, "grad_norm": 0.009342700242996216, "learning_rate": 5.8913975409115874e-05, "loss": 9.028498607221991e-05, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00009, "step": 256, "tokens/total": 8333624, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 117126} +{"epoch": 1.00390625, "grad_norm": 0.02443896234035492, "learning_rate": 5.8624705189843395e-05, "loss": 0.000154762456077151, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00015, "step": 257, "tokens/total": 8364976, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 117592} +{"epoch": 1.0078125, "grad_norm": 0.02811327576637268, "learning_rate": 5.833528413179249e-05, "loss": 0.00018668810662347823, "memory/device_reserved (GiB)": 66.62, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00019, "step": 258, "tokens/total": 8399568, "tokens/train_per_sec_per_gpu": 7.08, "tokens/trainable": 118030} +{"epoch": 1.01171875, "grad_norm": 0.8396246433258057, "learning_rate": 5.80457242789548e-05, "loss": 0.008129903115332127, "memory/device_reserved (GiB)": 66.62, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00816, "step": 259, "tokens/total": 8430616, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 118470} +{"epoch": 1.015625, "grad_norm": 0.33129721879959106, "learning_rate": 5.77560376810977e-05, "loss": 0.0015885740285739303, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00159, "step": 260, "tokens/total": 8462696, "tokens/train_per_sec_per_gpu": 10.2, "tokens/trainable": 118935} +{"epoch": 1.01953125, "grad_norm": 0.29269957542419434, "learning_rate": 5.7466236393263005e-05, "loss": 0.0013584428234025836, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00136, "step": 261, "tokens/total": 8496152, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 119381} +{"epoch": 1.0234375, "grad_norm": 0.4932962954044342, "learning_rate": 5.717633247526522e-05, "loss": 0.001074820989742875, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00108, "step": 262, "tokens/total": 8532912, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 119849} +{"epoch": 1.02734375, "grad_norm": 0.10343630611896515, "learning_rate": 5.688633799118971e-05, "loss": 0.00023106335720513016, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.00023, "step": 263, "tokens/total": 8565376, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 120319} +{"epoch": 1.03125, "grad_norm": 0.02749449759721756, "learning_rate": 5.659626500889066e-05, "loss": 0.00011860414815600961, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00012, "step": 264, "tokens/total": 8597640, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 120804} +{"epoch": 1.03515625, "grad_norm": 0.7379117012023926, "learning_rate": 5.6306125599488905e-05, "loss": 0.0005191419622860849, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.00052, "step": 265, "tokens/total": 8627312, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 121233} +{"epoch": 1.0390625, "grad_norm": 1.9827090501785278, "learning_rate": 5.601593183686955e-05, "loss": 0.006613577716052532, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.87, "memory/max_allocated (GiB)": 64.87, "ppl": 1.00664, "step": 266, "tokens/total": 8659208, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 121671} +{"epoch": 1.04296875, "grad_norm": 0.19947488605976105, "learning_rate": 5.572569579717961e-05, "loss": 0.000396621209802106, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.0004, "step": 267, "tokens/total": 8693088, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 122141} +{"epoch": 1.046875, "grad_norm": 0.33885982632637024, "learning_rate": 5.543542955832538e-05, "loss": 0.0009376686066389084, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00094, "step": 268, "tokens/total": 8725576, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 122556} +{"epoch": 1.05078125, "grad_norm": 13.569182395935059, "learning_rate": 5.514514519946986e-05, "loss": 0.005203161854296923, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00522, "step": 269, "tokens/total": 8755192, "tokens/train_per_sec_per_gpu": 10.5, "tokens/trainable": 122988} +{"epoch": 1.0546875, "grad_norm": 0.016712268814444542, "learning_rate": 5.485485480053015e-05, "loss": 8.312241698149592e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00008, "step": 270, "tokens/total": 8787704, "tokens/train_per_sec_per_gpu": 10.72, "tokens/trainable": 123478} +{"epoch": 1.05859375, "grad_norm": 0.09133990108966827, "learning_rate": 5.4564570441674645e-05, "loss": 0.00014693895354866982, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00015, "step": 271, "tokens/total": 8819768, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 123932} +{"epoch": 1.0625, "grad_norm": 0.02544749714434147, "learning_rate": 5.42743042028204e-05, "loss": 0.00011476147483335808, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00011, "step": 272, "tokens/total": 8853352, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 124428} +{"epoch": 1.06640625, "grad_norm": 1.0055087804794312, "learning_rate": 5.3984068163130464e-05, "loss": 0.002404655795544386, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00241, "step": 273, "tokens/total": 8886840, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 124866} +{"epoch": 1.0703125, "grad_norm": 0.04343826696276665, "learning_rate": 5.369387440051111e-05, "loss": 0.0001200392289320007, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00012, "step": 274, "tokens/total": 8920808, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 125324} +{"epoch": 1.07421875, "grad_norm": 0.5755695104598999, "learning_rate": 5.340373499110935e-05, "loss": 0.0011176727712154388, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00112, "step": 275, "tokens/total": 8955880, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 125813} +{"epoch": 1.078125, "grad_norm": 0.05220832675695419, "learning_rate": 5.3113662008810304e-05, "loss": 0.00027070732903666794, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.00027, "step": 276, "tokens/total": 8989520, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 126252} +{"epoch": 1.08203125, "grad_norm": 0.004359075799584389, "learning_rate": 5.282366752473479e-05, "loss": 5.229145972407423e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00005, "step": 277, "tokens/total": 9024680, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 126696} +{"epoch": 1.0859375, "grad_norm": 1.3400049209594727, "learning_rate": 5.2533763606737005e-05, "loss": 0.008588037453591824, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00863, "step": 278, "tokens/total": 9056208, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 127173} +{"epoch": 1.08984375, "grad_norm": 1.285782814025879, "learning_rate": 5.224396231890232e-05, "loss": 0.00624997541308403, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00627, "step": 279, "tokens/total": 9086376, "tokens/train_per_sec_per_gpu": 10.11, "tokens/trainable": 127637} +{"epoch": 1.09375, "grad_norm": 0.8648271560668945, "learning_rate": 5.195427572104522e-05, "loss": 0.001191414543427527, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00119, "step": 280, "tokens/total": 9118944, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 128112} +{"epoch": 1.09765625, "grad_norm": 0.14561288058757782, "learning_rate": 5.166471586820751e-05, "loss": 0.0004811005783267319, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00048, "step": 281, "tokens/total": 9152920, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 128584} +{"epoch": 1.1015625, "grad_norm": 0.9526474475860596, "learning_rate": 5.1375294810156615e-05, "loss": 0.006774467416107655, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.0068, "step": 282, "tokens/total": 9185728, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 129051} +{"epoch": 1.10546875, "grad_norm": 0.02713610976934433, "learning_rate": 5.1086024590884144e-05, "loss": 0.00011115351662738249, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00011, "step": 283, "tokens/total": 9217984, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 129494} +{"epoch": 1.109375, "grad_norm": 0.1837042272090912, "learning_rate": 5.079691724810461e-05, "loss": 0.0006210930878296494, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00062, "step": 284, "tokens/total": 9253160, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 129983} +{"epoch": 1.11328125, "grad_norm": 0.36699238419532776, "learning_rate": 5.0507984812754684e-05, "loss": 0.0012255854671820998, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00123, "step": 285, "tokens/total": 9286136, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 130405} +{"epoch": 1.1171875, "grad_norm": 1.6877139806747437, "learning_rate": 5.021923930849237e-05, "loss": 0.014387156814336777, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.01449, "step": 286, "tokens/total": 9318360, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 130829} +{"epoch": 1.12109375, "grad_norm": 0.0026027364656329155, "learning_rate": 4.99306927511967e-05, "loss": 2.3207194317365065e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00002, "step": 287, "tokens/total": 9350288, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 131287} +{"epoch": 1.125, "grad_norm": 1.4128438234329224, "learning_rate": 4.964235714846775e-05, "loss": 0.004098205361515284, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00411, "step": 288, "tokens/total": 9382104, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 131712} +{"epoch": 1.12890625, "grad_norm": 1.0153411626815796, "learning_rate": 4.9354244499126866e-05, "loss": 0.003979236353188753, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00399, "step": 289, "tokens/total": 9414392, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 132168} +{"epoch": 1.1328125, "grad_norm": 0.11439628899097443, "learning_rate": 4.90663667927174e-05, "loss": 0.00026301693287678063, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00026, "step": 290, "tokens/total": 9447112, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 132626} +{"epoch": 1.13671875, "grad_norm": 0.00475328741595149, "learning_rate": 4.877873600900581e-05, "loss": 6.238641799427569e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00006, "step": 291, "tokens/total": 9483400, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 133119} +{"epoch": 1.140625, "grad_norm": 1.2941824197769165, "learning_rate": 4.849136411748306e-05, "loss": 0.004191741347312927, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.48, "memory/max_allocated (GiB)": 64.48, "ppl": 1.0042, "step": 292, "tokens/total": 9513896, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 133550} +{"epoch": 1.14453125, "grad_norm": 4.628201007843018, "learning_rate": 4.8204263076866574e-05, "loss": 0.005302855744957924, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00532, "step": 293, "tokens/total": 9550184, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 133992} +{"epoch": 1.1484375, "grad_norm": 0.006370365619659424, "learning_rate": 4.791744483460251e-05, "loss": 7.593046757392585e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00008, "step": 294, "tokens/total": 9583920, "tokens/train_per_sec_per_gpu": 10.26, "tokens/trainable": 134467} +{"epoch": 1.15234375, "grad_norm": 0.04064241051673889, "learning_rate": 4.7630921326368736e-05, "loss": 0.0002397701609879732, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00024, "step": 295, "tokens/total": 9617288, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 134895} +{"epoch": 1.15625, "grad_norm": 0.9834859371185303, "learning_rate": 4.7344704475577916e-05, "loss": 0.01879529468715191, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 62.94, "memory/max_allocated (GiB)": 62.94, "ppl": 1.01897, "step": 296, "tokens/total": 9645600, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 135321} +{"epoch": 1.16015625, "grad_norm": 0.0820317193865776, "learning_rate": 4.705880619288153e-05, "loss": 0.0005983022856526077, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.0006, "step": 297, "tokens/total": 9679456, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 135765} +{"epoch": 1.1640625, "grad_norm": 0.7061283588409424, "learning_rate": 4.677323837567412e-05, "loss": 0.016034835949540138, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.01616, "step": 298, "tokens/total": 9711696, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 136196} +{"epoch": 1.16796875, "grad_norm": 0.38261088728904724, "learning_rate": 4.6488012907598146e-05, "loss": 0.0013890587724745274, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00139, "step": 299, "tokens/total": 9745792, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 136649} +{"epoch": 1.171875, "grad_norm": 0.3151722848415375, "learning_rate": 4.620314165804964e-05, "loss": 0.0021191046107560396, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00212, "step": 300, "tokens/total": 9780648, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 137128} +{"epoch": 1.17578125, "grad_norm": 0.3581860065460205, "learning_rate": 4.591863648168407e-05, "loss": 0.003537694690749049, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00354, "step": 301, "tokens/total": 9813032, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 137585} +{"epoch": 1.1796875, "grad_norm": 0.19425155222415924, "learning_rate": 4.5634509217923135e-05, "loss": 0.0021432784851640463, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00215, "step": 302, "tokens/total": 9845200, "tokens/train_per_sec_per_gpu": 10.2, "tokens/trainable": 138054} +{"epoch": 1.18359375, "grad_norm": 0.036635346710681915, "learning_rate": 4.535077169046201e-05, "loss": 0.0005273427814245224, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.29, "memory/max_allocated (GiB)": 65.29, "ppl": 1.00053, "step": 303, "tokens/total": 9878616, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 138559} +{"epoch": 1.1875, "grad_norm": 0.2716582715511322, "learning_rate": 4.506743570677743e-05, "loss": 0.005330198910087347, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00534, "step": 304, "tokens/total": 9910776, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 139050} +{"epoch": 1.19140625, "grad_norm": 0.5009113550186157, "learning_rate": 4.478451305763618e-05, "loss": 0.003075194777920842, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00308, "step": 305, "tokens/total": 9945480, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 139525} +{"epoch": 1.1953125, "grad_norm": 0.03638128936290741, "learning_rate": 4.450201551660454e-05, "loss": 0.00036832140176557004, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.00037, "step": 306, "tokens/total": 9975992, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 139969} +{"epoch": 1.19921875, "grad_norm": 0.4499591886997223, "learning_rate": 4.4219954839558276e-05, "loss": 0.0018163721542805433, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.00182, "step": 307, "tokens/total": 10006440, "tokens/train_per_sec_per_gpu": 10.07, "tokens/trainable": 140443} +{"epoch": 1.203125, "grad_norm": 0.0839148759841919, "learning_rate": 4.393834276419352e-05, "loss": 0.00025320108397863805, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00025, "step": 308, "tokens/total": 10042928, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 140926} +{"epoch": 1.20703125, "grad_norm": 0.34306907653808594, "learning_rate": 4.36571910095382e-05, "loss": 0.00045950314961373806, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00046, "step": 309, "tokens/total": 10075184, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 141344} +{"epoch": 1.2109375, "grad_norm": 0.7129852771759033, "learning_rate": 4.337651127546448e-05, "loss": 0.007620247546583414, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00765, "step": 310, "tokens/total": 10106656, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 141753} +{"epoch": 1.21484375, "grad_norm": 1.1848397254943848, "learning_rate": 4.3096315242201736e-05, "loss": 0.0066315652802586555, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.82, "memory/max_allocated (GiB)": 63.82, "ppl": 1.00665, "step": 311, "tokens/total": 10138536, "tokens/train_per_sec_per_gpu": 10.24, "tokens/trainable": 142219} +{"epoch": 1.21875, "grad_norm": 0.1721438616514206, "learning_rate": 4.2816614569850635e-05, "loss": 0.0013237359235063195, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.00132, "step": 312, "tokens/total": 10165768, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 142617} +{"epoch": 1.22265625, "grad_norm": 0.02221333235502243, "learning_rate": 4.2537420897897864e-05, "loss": 0.00022044390789233148, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00022, "step": 313, "tokens/total": 10201624, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 143103} +{"epoch": 1.2265625, "grad_norm": 0.010298279114067554, "learning_rate": 4.225874584473174e-05, "loss": 0.00015897968842182308, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00016, "step": 314, "tokens/total": 10233304, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 143535} +{"epoch": 1.23046875, "grad_norm": 0.23074400424957275, "learning_rate": 4.19806010071587e-05, "loss": 0.0005210707895457745, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00052, "step": 315, "tokens/total": 10267904, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 144009} +{"epoch": 1.234375, "grad_norm": 0.85922771692276, "learning_rate": 4.170299795992081e-05, "loss": 0.003409087425097823, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00341, "step": 316, "tokens/total": 10301280, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 144460} +{"epoch": 1.23828125, "grad_norm": 0.04251909255981445, "learning_rate": 4.142594825521398e-05, "loss": 0.0004202018608339131, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00042, "step": 317, "tokens/total": 10337072, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 144936} +{"epoch": 1.2421875, "grad_norm": 0.1362602561712265, "learning_rate": 4.114946342220728e-05, "loss": 0.0006967532681301236, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.69, "memory/max_allocated (GiB)": 64.69, "ppl": 1.0007, "step": 318, "tokens/total": 10368688, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 145375} +{"epoch": 1.24609375, "grad_norm": 0.0835605040192604, "learning_rate": 4.087355496656321e-05, "loss": 0.0006803751457482576, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00068, "step": 319, "tokens/total": 10400752, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 145849} +{"epoch": 1.25, "grad_norm": 0.032902516424655914, "learning_rate": 4.05982343699588e-05, "loss": 0.00045137963024899364, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00045, "step": 320, "tokens/total": 10432112, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 146318} +{"epoch": 1.25390625, "grad_norm": 0.23569433391094208, "learning_rate": 4.0323513089607876e-05, "loss": 0.0010734803508967161, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00107, "step": 321, "tokens/total": 10465472, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 146821} +{"epoch": 1.2578125, "grad_norm": 0.00929697323590517, "learning_rate": 4.004940255778431e-05, "loss": 9.59338722168468e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.0001, "step": 322, "tokens/total": 10496336, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 147285} +{"epoch": 1.26171875, "grad_norm": 0.026576383039355278, "learning_rate": 3.977591418134619e-05, "loss": 0.00010761887097032741, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00011, "step": 323, "tokens/total": 10525864, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 147729} +{"epoch": 1.265625, "grad_norm": 0.09984201937913895, "learning_rate": 3.95030593412612e-05, "loss": 0.0006289372686296701, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00063, "step": 324, "tokens/total": 10559680, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 148169} +{"epoch": 1.26953125, "grad_norm": 1.7944172620773315, "learning_rate": 3.923084939213296e-05, "loss": 0.007925794459879398, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00796, "step": 325, "tokens/total": 10595400, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 148630} +{"epoch": 1.2734375, "grad_norm": 0.2908463478088379, "learning_rate": 3.895929566172861e-05, "loss": 0.0015966303180903196, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.0016, "step": 326, "tokens/total": 10626680, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 149063} +{"epoch": 1.27734375, "grad_norm": 0.06452474743127823, "learning_rate": 3.868840945050728e-05, "loss": 0.00046314962673932314, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00046, "step": 327, "tokens/total": 10661784, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 149549} +{"epoch": 1.28125, "grad_norm": 0.3436139225959778, "learning_rate": 3.841820203114995e-05, "loss": 0.01303114928305149, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.31, "memory/max_allocated (GiB)": 65.31, "ppl": 1.01312, "step": 328, "tokens/total": 10696184, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 150015} +{"epoch": 1.28515625, "grad_norm": 0.0391339436173439, "learning_rate": 3.814868464809027e-05, "loss": 0.0003008765052072704, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.0003, "step": 329, "tokens/total": 10729376, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 150514} +{"epoch": 1.2890625, "grad_norm": 0.8768634796142578, "learning_rate": 3.787986851704667e-05, "loss": 0.0021059277933090925, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00211, "step": 330, "tokens/total": 10764824, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 151021} +{"epoch": 1.29296875, "grad_norm": 0.3632090091705322, "learning_rate": 3.7611764824555654e-05, "loss": 0.0026838453486561775, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00269, "step": 331, "tokens/total": 10797072, "tokens/train_per_sec_per_gpu": 10.1, "tokens/trainable": 151486} +{"epoch": 1.296875, "grad_norm": 0.008916088379919529, "learning_rate": 3.734438472750619e-05, "loss": 0.0001635467488085851, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00016, "step": 332, "tokens/total": 10829376, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 151956} +{"epoch": 1.30078125, "grad_norm": 0.43226441740989685, "learning_rate": 3.707773935267552e-05, "loss": 0.011501624248921871, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.01157, "step": 333, "tokens/total": 10863512, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 152442} +{"epoch": 1.3046875, "grad_norm": 0.031220335513353348, "learning_rate": 3.68118397962661e-05, "loss": 0.0002675588184501976, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00027, "step": 334, "tokens/total": 10894432, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 152890} +{"epoch": 1.30859375, "grad_norm": 0.22117915749549866, "learning_rate": 3.654669712344384e-05, "loss": 0.0024300990626215935, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.34, "memory/max_allocated (GiB)": 63.34, "ppl": 1.00243, "step": 335, "tokens/total": 10924848, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 153368} +{"epoch": 1.3125, "grad_norm": 0.033786602318286896, "learning_rate": 3.628232236787763e-05, "loss": 0.000241116649704054, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00024, "step": 336, "tokens/total": 10959024, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 153809} +{"epoch": 1.31640625, "grad_norm": 0.5212839245796204, "learning_rate": 3.6018726531280144e-05, "loss": 0.0032187856268137693, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00322, "step": 337, "tokens/total": 10991088, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 154258} +{"epoch": 1.3203125, "grad_norm": 0.6278730034828186, "learning_rate": 3.575592058295017e-05, "loss": 0.006671406328678131, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00669, "step": 338, "tokens/total": 11023904, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 154704} +{"epoch": 1.32421875, "grad_norm": 0.01923060044646263, "learning_rate": 3.549391545931585e-05, "loss": 0.0003934775304514915, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00039, "step": 339, "tokens/total": 11057680, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 155157} +{"epoch": 1.328125, "grad_norm": 0.17641186714172363, "learning_rate": 3.5232722063479914e-05, "loss": 0.0009610485285520554, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00096, "step": 340, "tokens/total": 11089800, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 155582} +{"epoch": 1.33203125, "grad_norm": 0.00958977360278368, "learning_rate": 3.49723512647657e-05, "loss": 0.00014163028390612453, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00014, "step": 341, "tokens/total": 11117960, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 156047} +{"epoch": 1.3359375, "grad_norm": 0.16896194219589233, "learning_rate": 3.471281389826491e-05, "loss": 0.001558612915687263, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00156, "step": 342, "tokens/total": 11151888, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 156500} +{"epoch": 1.33984375, "grad_norm": 1.4057817459106445, "learning_rate": 3.4454120764386764e-05, "loss": 0.002281642286106944, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.22, "memory/max_allocated (GiB)": 65.22, "ppl": 1.00228, "step": 343, "tokens/total": 11184568, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 156931} +{"epoch": 1.34375, "grad_norm": 0.11283671855926514, "learning_rate": 3.4196282628408526e-05, "loss": 0.0014319367473945022, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00143, "step": 344, "tokens/total": 11216400, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 157385} +{"epoch": 1.34765625, "grad_norm": 0.016772527247667313, "learning_rate": 3.3939310220027456e-05, "loss": 0.00012690597213804722, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00013, "step": 345, "tokens/total": 11250768, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 157817} +{"epoch": 1.3515625, "grad_norm": 0.0906834825873375, "learning_rate": 3.3683214232914404e-05, "loss": 0.0006454630056396127, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00065, "step": 346, "tokens/total": 11281808, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 158273} +{"epoch": 1.35546875, "grad_norm": 0.015147519297897816, "learning_rate": 3.342800532426873e-05, "loss": 0.00021627743262797594, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00022, "step": 347, "tokens/total": 11313968, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 158728} +{"epoch": 1.359375, "grad_norm": 0.025344062596559525, "learning_rate": 3.317369411437484e-05, "loss": 0.0001367651711916551, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00014, "step": 348, "tokens/total": 11347288, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 159168} +{"epoch": 1.36328125, "grad_norm": 0.19448839128017426, "learning_rate": 3.292029118616024e-05, "loss": 0.0010189003078266978, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00102, "step": 349, "tokens/total": 11377128, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 159592} +{"epoch": 1.3671875, "grad_norm": 0.08970776945352554, "learning_rate": 3.266780708475511e-05, "loss": 0.0005061401752755046, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00051, "step": 350, "tokens/total": 11409488, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 160093} +{"epoch": 1.37109375, "grad_norm": 0.09834708273410797, "learning_rate": 3.241625231705354e-05, "loss": 0.0008442765683867037, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00084, "step": 351, "tokens/total": 11442040, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 160534} +{"epoch": 1.375, "grad_norm": 0.08525083214044571, "learning_rate": 3.216563735127618e-05, "loss": 0.00032289032242260873, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00032, "step": 352, "tokens/total": 11478312, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 161009} +{"epoch": 1.37890625, "grad_norm": 0.03515458106994629, "learning_rate": 3.191597261653475e-05, "loss": 0.0004699307610280812, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.93, "memory/max_allocated (GiB)": 63.93, "ppl": 1.00047, "step": 353, "tokens/total": 11509008, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 161450} +{"epoch": 1.3828125, "grad_norm": 0.24330101907253265, "learning_rate": 3.166726850239794e-05, "loss": 0.0014402058441191912, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00144, "step": 354, "tokens/total": 11541264, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 161900} +{"epoch": 1.38671875, "grad_norm": 0.0061284638941287994, "learning_rate": 3.141953535845912e-05, "loss": 6.842465518275276e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00007, "step": 355, "tokens/total": 11573072, "tokens/train_per_sec_per_gpu": 10.25, "tokens/trainable": 162358} +{"epoch": 1.390625, "grad_norm": 0.06574694067239761, "learning_rate": 3.11727834939056e-05, "loss": 0.00046965936780907214, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00047, "step": 356, "tokens/total": 11605224, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 162845} +{"epoch": 1.39453125, "grad_norm": 0.6971727609634399, "learning_rate": 3.092702317708967e-05, "loss": 0.004288456868380308, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.0043, "step": 357, "tokens/total": 11637512, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 163299} +{"epoch": 1.3984375, "grad_norm": 0.044177260249853134, "learning_rate": 3.0682264635101276e-05, "loss": 0.00015348786837421358, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00015, "step": 358, "tokens/total": 11671280, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 163774} +{"epoch": 1.40234375, "grad_norm": 0.0060674711130559444, "learning_rate": 3.0438518053342407e-05, "loss": 6.998746539466083e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00007, "step": 359, "tokens/total": 11702008, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 164199} +{"epoch": 1.40625, "grad_norm": 0.035945381969213486, "learning_rate": 3.0195793575103266e-05, "loss": 0.00021614794968627393, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00022, "step": 360, "tokens/total": 11732032, "tokens/train_per_sec_per_gpu": 10.17, "tokens/trainable": 164667} +{"epoch": 1.41015625, "grad_norm": 0.1576882153749466, "learning_rate": 2.9954101301140146e-05, "loss": 0.0017771520651876926, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00178, "step": 361, "tokens/total": 11767472, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.01889018528163433, "learning_rate": 2.9713451289255123e-05, "loss": 0.00011795755563070998, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00012, "step": 362, "tokens/total": 11798432, "tokens/train_per_sec_per_gpu": 10.63, "tokens/trainable": 165575} +{"epoch": 1.41796875, "grad_norm": 1.0768104791641235, "learning_rate": 2.9473853553877484e-05, "loss": 0.0007586824358440936, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00076, "step": 363, "tokens/total": 11833144, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 166060} +{"epoch": 1.421875, "grad_norm": 0.016165707260370255, "learning_rate": 2.9235318065647e-05, "loss": 8.31565703265369e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.00008, "step": 364, "tokens/total": 11865856, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 166478} +{"epoch": 1.42578125, "grad_norm": 2.225752115249634, "learning_rate": 2.8997854750998964e-05, "loss": 0.02190108597278595, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.02214, "step": 365, "tokens/total": 11900400, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 166941} +{"epoch": 1.4296875, "grad_norm": 0.00871474388986826, "learning_rate": 2.8761473491751258e-05, "loss": 5.8401692513143644e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.49, "memory/max_allocated (GiB)": 63.49, "ppl": 1.00006, "step": 366, "tokens/total": 11931368, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 167381} +{"epoch": 1.43359375, "grad_norm": 0.12804396450519562, "learning_rate": 2.8526184124692883e-05, "loss": 0.0003760272520594299, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00038, "step": 367, "tokens/total": 11963320, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 167830} +{"epoch": 1.4375, "grad_norm": 0.02729222923517227, "learning_rate": 2.829199644117484e-05, "loss": 0.0001787976361811161, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00018, "step": 368, "tokens/total": 11997536, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 168266} +{"epoch": 1.44140625, "grad_norm": 0.029290271922945976, "learning_rate": 2.8058920186702553e-05, "loss": 0.00018293173343408853, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00018, "step": 369, "tokens/total": 12029528, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 168716} +{"epoch": 1.4453125, "grad_norm": 0.026563910767436028, "learning_rate": 2.782696506053033e-05, "loss": 0.00014041246322449297, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00014, "step": 370, "tokens/total": 12062504, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 169176} +{"epoch": 1.44921875, "grad_norm": 0.057314418256282806, "learning_rate": 2.7596140715257824e-05, "loss": 0.00025892426492646337, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.78, "memory/max_allocated (GiB)": 65.78, "ppl": 1.00026, "step": 371, "tokens/total": 12097376, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 169591} +{"epoch": 1.453125, "grad_norm": 0.011745696887373924, "learning_rate": 2.7366456756428184e-05, "loss": 0.00011571808136068285, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00012, "step": 372, "tokens/total": 12131920, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 170041} +{"epoch": 1.45703125, "grad_norm": 0.02187616564333439, "learning_rate": 2.7137922742128486e-05, "loss": 0.0001920807990245521, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00019, "step": 373, "tokens/total": 12164528, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 170516} +{"epoch": 1.4609375, "grad_norm": 0.04599076882004738, "learning_rate": 2.691054818259188e-05, "loss": 0.00032775814179331064, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.00033, "step": 374, "tokens/total": 12197376, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 170952} +{"epoch": 1.46484375, "grad_norm": 0.36965301632881165, "learning_rate": 2.6684342539801933e-05, "loss": 0.005097207613289356, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00511, "step": 375, "tokens/total": 12233104, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 171424} +{"epoch": 1.46875, "grad_norm": 0.010902602225542068, "learning_rate": 2.645931522709877e-05, "loss": 6.42065133433789e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.69, "memory/max_allocated (GiB)": 65.69, "ppl": 1.00006, "step": 376, "tokens/total": 12269224, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 171884} +{"epoch": 1.47265625, "grad_norm": 0.002296088496223092, "learning_rate": 2.6235475608787365e-05, "loss": 3.076395296375267e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00003, "step": 377, "tokens/total": 12302952, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 172404} +{"epoch": 1.4765625, "grad_norm": 1.2525224685668945, "learning_rate": 2.6012832999747916e-05, "loss": 0.002135735470801592, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00214, "step": 378, "tokens/total": 12338424, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 172871} +{"epoch": 1.48046875, "grad_norm": 0.03538252040743828, "learning_rate": 2.579139666504821e-05, "loss": 0.00011570197966648266, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00012, "step": 379, "tokens/total": 12369144, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 173329} +{"epoch": 1.484375, "grad_norm": 0.11087080091238022, "learning_rate": 2.557117581955798e-05, "loss": 0.0009777526138350368, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00098, "step": 380, "tokens/total": 12403352, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 173827} +{"epoch": 1.48828125, "grad_norm": 0.02547728270292282, "learning_rate": 2.5352179627565532e-05, "loss": 5.52158999198582e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00006, "step": 381, "tokens/total": 12441056, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 174287} +{"epoch": 1.4921875, "grad_norm": 1.878172755241394, "learning_rate": 2.5134417202396277e-05, "loss": 0.0011405411642044783, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.00114, "step": 382, "tokens/total": 12474080, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 174731} +{"epoch": 1.49609375, "grad_norm": 0.03374815732240677, "learning_rate": 2.491789760603361e-05, "loss": 0.0001739174040267244, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00017, "step": 383, "tokens/total": 12508992, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 175183} +{"epoch": 1.5, "grad_norm": 0.006681138649582863, "learning_rate": 2.4702629848741764e-05, "loss": 6.378235411830246e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00006, "step": 384, "tokens/total": 12540896, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 175655} +{"epoch": 1.50390625, "grad_norm": 0.015464171767234802, "learning_rate": 2.4488622888690785e-05, "loss": 0.0001839900214690715, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00018, "step": 385, "tokens/total": 12568416, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 176085} +{"epoch": 1.5078125, "grad_norm": 0.0028165895491838455, "learning_rate": 2.427588563158384e-05, "loss": 3.332804044475779e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00003, "step": 386, "tokens/total": 12599792, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 176549} +{"epoch": 1.51171875, "grad_norm": 0.007678838912397623, "learning_rate": 2.406442693028651e-05, "loss": 5.850574962096289e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00006, "step": 387, "tokens/total": 12631224, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 176995} +{"epoch": 1.515625, "grad_norm": 0.8345806002616882, "learning_rate": 2.3854255584458547e-05, "loss": 0.0011391304433345795, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00114, "step": 388, "tokens/total": 12667240, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 177468} +{"epoch": 1.51953125, "grad_norm": 0.029854783788323402, "learning_rate": 2.3645380340187508e-05, "loss": 0.0002834236074704677, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.54, "memory/max_allocated (GiB)": 65.54, "ppl": 1.00028, "step": 389, "tokens/total": 12700488, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 177961} +{"epoch": 1.5234375, "grad_norm": 0.010011865757405758, "learning_rate": 2.3437809889624914e-05, "loss": 6.441816367441788e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.00006, "step": 390, "tokens/total": 12734528, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 178417} +{"epoch": 1.52734375, "grad_norm": 0.038292739540338516, "learning_rate": 2.3231552870624487e-05, "loss": 0.0003504286869429052, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00035, "step": 391, "tokens/total": 12767152, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 178862} +{"epoch": 1.53125, "grad_norm": 0.4345826506614685, "learning_rate": 2.3026617866382657e-05, "loss": 0.001532484428025782, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.00153, "step": 392, "tokens/total": 12798168, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 179320} +{"epoch": 1.53515625, "grad_norm": 0.0027441487181931734, "learning_rate": 2.2823013405081507e-05, "loss": 4.587811781675555e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00005, "step": 393, "tokens/total": 12828896, "tokens/train_per_sec_per_gpu": 10.05, "tokens/trainable": 179758} +{"epoch": 1.5390625, "grad_norm": 0.1437571495771408, "learning_rate": 2.2620747959533722e-05, "loss": 0.0009229946881532669, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00092, "step": 394, "tokens/total": 12860024, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 180211} +{"epoch": 1.54296875, "grad_norm": 0.6297142505645752, "learning_rate": 2.2419829946830123e-05, "loss": 0.007099592592567205, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00712, "step": 395, "tokens/total": 12891472, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 180637} +{"epoch": 1.546875, "grad_norm": 0.43455615639686584, "learning_rate": 2.2220267727989325e-05, "loss": 0.00210554338991642, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00211, "step": 396, "tokens/total": 12925648, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 181142} +{"epoch": 1.55078125, "grad_norm": 0.007650100626051426, "learning_rate": 2.202206960760984e-05, "loss": 5.731343844672665e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.77, "memory/max_allocated (GiB)": 64.77, "ppl": 1.00006, "step": 397, "tokens/total": 12959200, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 181597} +{"epoch": 1.5546875, "grad_norm": 0.0057309092953801155, "learning_rate": 2.182524383352446e-05, "loss": 6.165716331452131e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00006, "step": 398, "tokens/total": 12989624, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 182030} +{"epoch": 1.55859375, "grad_norm": 0.003670239355415106, "learning_rate": 2.1629798596457056e-05, "loss": 2.2419995730160736e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 62.97, "memory/max_allocated (GiB)": 62.97, "ppl": 1.00002, "step": 399, "tokens/total": 13019040, "tokens/train_per_sec_per_gpu": 10.49, "tokens/trainable": 182496} +{"epoch": 1.5625, "grad_norm": 0.011149559170007706, "learning_rate": 2.1435742029681725e-05, "loss": 6.206068792380393e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00006, "step": 400, "tokens/total": 13051808, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 182963} +{"epoch": 1.56640625, "grad_norm": 0.006331074051558971, "learning_rate": 2.124308220868431e-05, "loss": 4.8363399400841445e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00005, "step": 401, "tokens/total": 13085520, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 183416} +{"epoch": 1.5703125, "grad_norm": 0.0009965641656890512, "learning_rate": 2.105182715082638e-05, "loss": 1.793094452295918e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00002, "step": 402, "tokens/total": 13116224, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 183857} +{"epoch": 1.57421875, "grad_norm": 0.0250376146286726, "learning_rate": 2.0861984815011552e-05, "loss": 9.292493632528931e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00009, "step": 403, "tokens/total": 13149984, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 184296} +{"epoch": 1.578125, "grad_norm": 1.3674702644348145, "learning_rate": 2.0673563101354323e-05, "loss": 0.0060591986402869225, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00608, "step": 404, "tokens/total": 13185488, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 184719} +{"epoch": 1.58203125, "grad_norm": 0.018936149775981903, "learning_rate": 2.0486569850851317e-05, "loss": 0.00013404343917500228, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00013, "step": 405, "tokens/total": 13216320, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 185172} +{"epoch": 1.5859375, "grad_norm": 0.00816091988235712, "learning_rate": 2.0301012845054956e-05, "loss": 4.222044663038105e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00004, "step": 406, "tokens/total": 13248136, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 185646} +{"epoch": 1.58984375, "grad_norm": 1.607089877128601, "learning_rate": 2.011689980574966e-05, "loss": 0.0007039025076664984, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.0007, "step": 407, "tokens/total": 13280640, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 186129} +{"epoch": 1.59375, "grad_norm": 0.07150515168905258, "learning_rate": 1.993423839463052e-05, "loss": 0.0005046170554123819, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.0005, "step": 408, "tokens/total": 13314160, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 186610} +{"epoch": 1.59765625, "grad_norm": 0.07613608986139297, "learning_rate": 1.975303621298445e-05, "loss": 0.0004145151760894805, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.51, "memory/max_allocated (GiB)": 63.51, "ppl": 1.00041, "step": 409, "tokens/total": 13345024, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 187066} +{"epoch": 1.6015625, "grad_norm": 0.03943008556962013, "learning_rate": 1.957330080137385e-05, "loss": 5.0424747314536944e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00005, "step": 410, "tokens/total": 13377248, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 187567} +{"epoch": 1.60546875, "grad_norm": 0.3087927997112274, "learning_rate": 1.9395039639322864e-05, "loss": 0.007190424017608166, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00722, "step": 411, "tokens/total": 13410992, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 188056} +{"epoch": 1.609375, "grad_norm": 0.004602437373250723, "learning_rate": 1.9218260145006073e-05, "loss": 3.952308543375693e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00004, "step": 412, "tokens/total": 13442728, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 188502} +{"epoch": 1.61328125, "grad_norm": 0.13568012416362762, "learning_rate": 1.904296967493982e-05, "loss": 0.0008235753048211336, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00082, "step": 413, "tokens/total": 13474824, "tokens/train_per_sec_per_gpu": 10.24, "tokens/trainable": 188995} +{"epoch": 1.6171875, "grad_norm": 0.5802817344665527, "learning_rate": 1.8869175523676064e-05, "loss": 0.004138801246881485, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00415, "step": 414, "tokens/total": 13506528, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 189483} +{"epoch": 1.62109375, "grad_norm": 0.6488091349601746, "learning_rate": 1.869688492349885e-05, "loss": 0.006951813120394945, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00698, "step": 415, "tokens/total": 13537792, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 189935} +{"epoch": 1.625, "grad_norm": 0.21613185107707977, "learning_rate": 1.85261050441233e-05, "loss": 0.0006932287360541523, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.00069, "step": 416, "tokens/total": 13573136, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 190399} +{"epoch": 1.62890625, "grad_norm": 0.022808361798524857, "learning_rate": 1.8356842992397304e-05, "loss": 0.00021668207773473114, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00022, "step": 417, "tokens/total": 13607240, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 190862} +{"epoch": 1.6328125, "grad_norm": 0.01807907223701477, "learning_rate": 1.8189105812005714e-05, "loss": 1.3246792150312103e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00001, "step": 418, "tokens/total": 13638920, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 191294} +{"epoch": 1.63671875, "grad_norm": 0.03246168792247772, "learning_rate": 1.802290048317732e-05, "loss": 8.610126678831875e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00009, "step": 419, "tokens/total": 13672320, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 191764} +{"epoch": 1.640625, "grad_norm": 0.5584520101547241, "learning_rate": 1.785823392239424e-05, "loss": 0.0013976730406284332, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.0014, "step": 420, "tokens/total": 13703312, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 192196} +{"epoch": 1.64453125, "grad_norm": 0.009548281319439411, "learning_rate": 1.7695112982104225e-05, "loss": 7.367815123870969e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00007, "step": 421, "tokens/total": 13734440, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 192612} +{"epoch": 1.6484375, "grad_norm": 0.001974484883248806, "learning_rate": 1.7533544450435433e-05, "loss": 2.4859196855686605e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.62, "memory/max_allocated (GiB)": 64.62, "ppl": 1.00002, "step": 422, "tokens/total": 13766944, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 193059} +{"epoch": 1.65234375, "grad_norm": 0.07777103036642075, "learning_rate": 1.7373535050913946e-05, "loss": 0.00038501113886013627, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00039, "step": 423, "tokens/total": 13797120, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 193466} +{"epoch": 1.65625, "grad_norm": 0.18583565950393677, "learning_rate": 1.721509144218405e-05, "loss": 0.0008129706839099526, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00081, "step": 424, "tokens/total": 13828040, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 193910} +{"epoch": 1.66015625, "grad_norm": 0.0024933016393333673, "learning_rate": 1.705822021773101e-05, "loss": 3.014743560925126e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.06, "memory/max_allocated (GiB)": 63.06, "ppl": 1.00003, "step": 425, "tokens/total": 13856328, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 194314} +{"epoch": 1.6640625, "grad_norm": 0.01805764250457287, "learning_rate": 1.69029279056068e-05, "loss": 0.00010390602983534336, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0001, "step": 426, "tokens/total": 13891312, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 194782} +{"epoch": 1.66796875, "grad_norm": 0.9655917286872864, "learning_rate": 1.6749220968158415e-05, "loss": 0.010807466693222523, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.01087, "step": 427, "tokens/total": 13925272, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 195254} +{"epoch": 1.671875, "grad_norm": 0.09382237493991852, "learning_rate": 1.659710580175893e-05, "loss": 0.0002578616258688271, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.53, "memory/max_allocated (GiB)": 63.53, "ppl": 1.00026, "step": 428, "tokens/total": 13955752, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 195702} +{"epoch": 1.67578125, "grad_norm": 0.262398898601532, "learning_rate": 1.644658873654133e-05, "loss": 0.0018706874689087272, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00187, "step": 429, "tokens/total": 13988000, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 196164} +{"epoch": 1.6796875, "grad_norm": 0.06854787468910217, "learning_rate": 1.629767603613508e-05, "loss": 0.00024626468075439334, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00025, "step": 430, "tokens/total": 14018712, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 196604} +{"epoch": 1.68359375, "grad_norm": 0.027876393869519234, "learning_rate": 1.615037389740547e-05, "loss": 0.00011589588393690065, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00012, "step": 431, "tokens/total": 14051992, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 197070} +{"epoch": 1.6875, "grad_norm": 0.4457194209098816, "learning_rate": 1.600468845019576e-05, "loss": 0.00043701616232283413, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.36, "memory/max_allocated (GiB)": 63.36, "ppl": 1.00044, "step": 432, "tokens/total": 14082624, "tokens/train_per_sec_per_gpu": 10.55, "tokens/trainable": 197552} +{"epoch": 1.69140625, "grad_norm": 0.18831276893615723, "learning_rate": 1.5860625757072092e-05, "loss": 0.000531736935954541, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00053, "step": 433, "tokens/total": 14113304, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 197990} +{"epoch": 1.6953125, "grad_norm": 0.022507231682538986, "learning_rate": 1.571819181307116e-05, "loss": 0.00010846667282748967, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00011, "step": 434, "tokens/total": 14149592, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 198513} +{"epoch": 1.69921875, "grad_norm": 0.007663256488740444, "learning_rate": 1.557739254545075e-05, "loss": 7.265490421559662e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00007, "step": 435, "tokens/total": 14183584, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 198979} +{"epoch": 1.703125, "grad_norm": 0.16852717101573944, "learning_rate": 1.543823381344311e-05, "loss": 8.372097363462672e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00008, "step": 436, "tokens/total": 14213688, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 199443} +{"epoch": 1.70703125, "grad_norm": 0.04922354593873024, "learning_rate": 1.5300721408011114e-05, "loss": 0.0002596485719550401, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.63, "memory/max_allocated (GiB)": 65.63, "ppl": 1.00026, "step": 437, "tokens/total": 14246480, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 199899} +{"epoch": 1.7109375, "grad_norm": 0.03223848715424538, "learning_rate": 1.5164861051607254e-05, "loss": 0.00014205690240487456, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00014, "step": 438, "tokens/total": 14278320, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 200368} +{"epoch": 1.71484375, "grad_norm": 0.800186812877655, "learning_rate": 1.5030658397935521e-05, "loss": 0.00457305321469903, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00458, "step": 439, "tokens/total": 14313016, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 200819} +{"epoch": 1.71875, "grad_norm": 0.0040497747249901295, "learning_rate": 1.4898119031716104e-05, "loss": 3.818488767137751e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00004, "step": 440, "tokens/total": 14343352, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 201275} +{"epoch": 1.72265625, "grad_norm": 0.01386839896440506, "learning_rate": 1.476724846845306e-05, "loss": 9.710984886623919e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.0001, "step": 441, "tokens/total": 14377736, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 201699} +{"epoch": 1.7265625, "grad_norm": 0.23640067875385284, "learning_rate": 1.463805215420471e-05, "loss": 0.0009450520155951381, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00095, "step": 442, "tokens/total": 14415056, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 202200} +{"epoch": 1.73046875, "grad_norm": 1.0797234773635864, "learning_rate": 1.451053546535705e-05, "loss": 0.003660322166979313, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00367, "step": 443, "tokens/total": 14449544, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 202640} +{"epoch": 1.734375, "grad_norm": 0.004176172893494368, "learning_rate": 1.438470370840001e-05, "loss": 4.151229950366542e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00004, "step": 444, "tokens/total": 14483472, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 203122} +{"epoch": 1.73828125, "grad_norm": 0.004665129818022251, "learning_rate": 1.4260562119706606e-05, "loss": 6.698090874124318e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.19, "memory/max_allocated (GiB)": 65.19, "ppl": 1.00007, "step": 445, "tokens/total": 14517648, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 203587} +{"epoch": 1.7421875, "grad_norm": 0.06207146868109703, "learning_rate": 1.413811586531508e-05, "loss": 0.0002513908548280597, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00025, "step": 446, "tokens/total": 14553056, "tokens/train_per_sec_per_gpu": 6.88, "tokens/trainable": 204024} +{"epoch": 1.74609375, "grad_norm": 0.031214995309710503, "learning_rate": 1.4017370040713884e-05, "loss": 0.000154736393596977, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.00015, "step": 447, "tokens/total": 14584280, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 204465} +{"epoch": 1.75, "grad_norm": 0.39732182025909424, "learning_rate": 1.3898329670629645e-05, "loss": 0.0018393582431599498, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00184, "step": 448, "tokens/total": 14618320, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 204923} +{"epoch": 1.75390625, "grad_norm": 0.007082028780132532, "learning_rate": 1.3780999708818058e-05, "loss": 8.461073593934998e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00008, "step": 449, "tokens/total": 14651264, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 205388} +{"epoch": 1.7578125, "grad_norm": 0.0204464178532362, "learning_rate": 1.3665385037857758e-05, "loss": 0.00018044793978333473, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00018, "step": 450, "tokens/total": 14682896, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 205873} +{"epoch": 1.76171875, "grad_norm": 0.30358874797821045, "learning_rate": 1.3551490468947126e-05, "loss": 0.0008883342961780727, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00089, "step": 451, "tokens/total": 14715272, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 206346} +{"epoch": 1.765625, "grad_norm": 0.1481655091047287, "learning_rate": 1.3439320741704075e-05, "loss": 0.0005654762499034405, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00057, "step": 452, "tokens/total": 14744840, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 206776} +{"epoch": 1.76953125, "grad_norm": 0.019264763221144676, "learning_rate": 1.3328880523968808e-05, "loss": 0.00016305921599268913, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00016, "step": 453, "tokens/total": 14778008, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 207233} +{"epoch": 1.7734375, "grad_norm": 0.03416196629405022, "learning_rate": 1.3220174411609587e-05, "loss": 0.00014827180712018162, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00015, "step": 454, "tokens/total": 14808472, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 207695} +{"epoch": 1.77734375, "grad_norm": 0.0018834472866728902, "learning_rate": 1.3113206928331471e-05, "loss": 2.2917374735698104e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00002, "step": 455, "tokens/total": 14841048, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 208144} +{"epoch": 1.78125, "grad_norm": 0.04277557507157326, "learning_rate": 1.300798252548806e-05, "loss": 0.0003264089173171669, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00033, "step": 456, "tokens/total": 14876904, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 208602} +{"epoch": 1.78515625, "grad_norm": 0.011837261728942394, "learning_rate": 1.2904505581896265e-05, "loss": 0.00012540111492853612, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.58, "memory/max_allocated (GiB)": 64.58, "ppl": 1.00013, "step": 457, "tokens/total": 14909512, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 209040} +{"epoch": 1.7890625, "grad_norm": 0.006280343513935804, "learning_rate": 1.2802780403654082e-05, "loss": 5.652304389514029e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00006, "step": 458, "tokens/total": 14942048, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 209501} +{"epoch": 1.79296875, "grad_norm": 0.1361394226551056, "learning_rate": 1.2702811223961408e-05, "loss": 0.0004055551835335791, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.00041, "step": 459, "tokens/total": 14977032, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 209993} +{"epoch": 1.796875, "grad_norm": 2.585758924484253, "learning_rate": 1.2604602202943861e-05, "loss": 0.013203212060034275, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01329, "step": 460, "tokens/total": 15011904, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 210455} +{"epoch": 1.80078125, "grad_norm": 0.1731339693069458, "learning_rate": 1.2508157427479686e-05, "loss": 0.002158514456823468, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00216, "step": 461, "tokens/total": 15045504, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 210960} +{"epoch": 1.8046875, "grad_norm": 0.3000352680683136, "learning_rate": 1.2413480911029655e-05, "loss": 0.0008022189722396433, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.15, "memory/max_allocated (GiB)": 66.15, "ppl": 1.0008, "step": 462, "tokens/total": 15078536, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 211405} +{"epoch": 1.80859375, "grad_norm": 0.020964309573173523, "learning_rate": 1.2320576593470082e-05, "loss": 9.029747889144346e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00009, "step": 463, "tokens/total": 15111640, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 211881} +{"epoch": 1.8125, "grad_norm": 0.09348297864198685, "learning_rate": 1.2229448340928828e-05, "loss": 0.00029780637123622, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.0003, "step": 464, "tokens/total": 15142608, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 212338} +{"epoch": 1.81640625, "grad_norm": 0.03233993053436279, "learning_rate": 1.2140099945624458e-05, "loss": 0.00019991594308521599, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.0002, "step": 465, "tokens/total": 15175048, "tokens/train_per_sec_per_gpu": 10.22, "tokens/trainable": 212803} +{"epoch": 1.8203125, "grad_norm": 0.012721189297735691, "learning_rate": 1.205253512570841e-05, "loss": 0.00010314340761397034, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.0001, "step": 466, "tokens/total": 15205776, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 213215} +{"epoch": 1.82421875, "grad_norm": 0.005304882302880287, "learning_rate": 1.1966757525110255e-05, "loss": 5.1048406021436676e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.00005, "step": 467, "tokens/total": 15239240, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 213675} +{"epoch": 1.828125, "grad_norm": 0.6377570629119873, "learning_rate": 1.1882770713386095e-05, "loss": 0.00250491825863719, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00251, "step": 468, "tokens/total": 15273696, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 214169} +{"epoch": 1.83203125, "grad_norm": 0.0012189893750473857, "learning_rate": 1.180057818556998e-05, "loss": 1.6198093362618238e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00002, "step": 469, "tokens/total": 15307632, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 214624} +{"epoch": 1.8359375, "grad_norm": 0.031984973698854446, "learning_rate": 1.1720183362028494e-05, "loss": 0.00010143428517039865, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.0001, "step": 470, "tokens/total": 15339960, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 215087} +{"epoch": 1.83984375, "grad_norm": 0.07463504374027252, "learning_rate": 1.1641589588318387e-05, "loss": 0.00010563002433627844, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00011, "step": 471, "tokens/total": 15373640, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 215553} +{"epoch": 1.84375, "grad_norm": 0.002340397797524929, "learning_rate": 1.1564800135047418e-05, "loss": 3.7823683669557795e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00004, "step": 472, "tokens/total": 15406304, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 216018} +{"epoch": 1.84765625, "grad_norm": 0.007586154155433178, "learning_rate": 1.148981819773816e-05, "loss": 3.206761903129518e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00003, "step": 473, "tokens/total": 15440128, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 216477} +{"epoch": 1.8515625, "grad_norm": 0.006652539595961571, "learning_rate": 1.1416646896695086e-05, "loss": 7.877144525991753e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00008, "step": 474, "tokens/total": 15471600, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 216949} +{"epoch": 1.85546875, "grad_norm": 0.013417829759418964, "learning_rate": 1.1345289276874717e-05, "loss": 0.0001094998515327461, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00011, "step": 475, "tokens/total": 15507016, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 217408} +{"epoch": 1.859375, "grad_norm": 0.884009838104248, "learning_rate": 1.1275748307758873e-05, "loss": 0.005095303989946842, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00511, "step": 476, "tokens/total": 15542600, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 217854} +{"epoch": 1.86328125, "grad_norm": 0.006295948754996061, "learning_rate": 1.1208026883231147e-05, "loss": 3.447546259849332e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00003, "step": 477, "tokens/total": 15577320, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 218337} +{"epoch": 1.8671875, "grad_norm": 0.22499863803386688, "learning_rate": 1.1142127821456433e-05, "loss": 0.001126810791902244, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00113, "step": 478, "tokens/total": 15608840, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 218788} +{"epoch": 1.87109375, "grad_norm": 0.02108529582619667, "learning_rate": 1.1078053864763674e-05, "loss": 6.856989057268947e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00007, "step": 479, "tokens/total": 15640160, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 219219} +{"epoch": 1.875, "grad_norm": 1.036549687385559, "learning_rate": 1.1015807679531756e-05, "loss": 0.006250226870179176, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00627, "step": 480, "tokens/total": 15673920, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 219676} +{"epoch": 1.87890625, "grad_norm": 0.003268637927249074, "learning_rate": 1.0955391856078528e-05, "loss": 4.544599505607039e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00005, "step": 481, "tokens/total": 15705512, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 220125} +{"epoch": 1.8828125, "grad_norm": 0.057001449167728424, "learning_rate": 1.0896808908553007e-05, "loss": 0.0004545625124592334, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00045, "step": 482, "tokens/total": 15742096, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 220615} +{"epoch": 1.88671875, "grad_norm": 0.0035297409631311893, "learning_rate": 1.0840061274830763e-05, "loss": 2.9976125006214716e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00003, "step": 483, "tokens/total": 15773064, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 221081} +{"epoch": 1.890625, "grad_norm": 0.07443953305482864, "learning_rate": 1.0785151316412473e-05, "loss": 0.00019870250253006816, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.0002, "step": 484, "tokens/total": 15805368, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 221546} +{"epoch": 1.89453125, "grad_norm": 0.0257632527500391, "learning_rate": 1.0732081318325639e-05, "loss": 0.00019160678493790329, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.57, "memory/max_allocated (GiB)": 65.57, "ppl": 1.00019, "step": 485, "tokens/total": 15839480, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 222014} +{"epoch": 1.8984375, "grad_norm": 0.008886975236237049, "learning_rate": 1.0680853489029501e-05, "loss": 8.098933903966099e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00008, "step": 486, "tokens/total": 15870672, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 222446} +{"epoch": 1.90234375, "grad_norm": 0.12909819185733795, "learning_rate": 1.0631469960323152e-05, "loss": 0.00030342486570589244, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.0003, "step": 487, "tokens/total": 15902784, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 222899} +{"epoch": 1.90625, "grad_norm": 0.05461721867322922, "learning_rate": 1.0583932787256783e-05, "loss": 8.74276302056387e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00009, "step": 488, "tokens/total": 15935456, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 223333} +{"epoch": 1.91015625, "grad_norm": 1.2341519594192505, "learning_rate": 1.0538243948046206e-05, "loss": 0.001528753899037838, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00153, "step": 489, "tokens/total": 15967960, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 223780} +{"epoch": 1.9140625, "grad_norm": 1.9648436307907104, "learning_rate": 1.0494405343990523e-05, "loss": 0.008805835619568825, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00884, "step": 490, "tokens/total": 16003904, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 224253} +{"epoch": 1.91796875, "grad_norm": 0.003861208911985159, "learning_rate": 1.0452418799392985e-05, "loss": 3.892180393449962e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00004, "step": 491, "tokens/total": 16035816, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 224699} +{"epoch": 1.921875, "grad_norm": 0.02261369302868843, "learning_rate": 1.0412286061485102e-05, "loss": 9.507147478871047e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.26, "memory/max_allocated (GiB)": 65.26, "ppl": 1.0001, "step": 492, "tokens/total": 16066880, "tokens/train_per_sec_per_gpu": 10.57, "tokens/trainable": 225147} +{"epoch": 1.92578125, "grad_norm": 0.043985266238451004, "learning_rate": 1.03740088003539e-05, "loss": 7.595459464937449e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00008, "step": 493, "tokens/total": 16101168, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 225596} +{"epoch": 1.9296875, "grad_norm": 0.07479332387447357, "learning_rate": 1.0337588608872463e-05, "loss": 0.0003947427321691066, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00039, "step": 494, "tokens/total": 16135208, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 226037} +{"epoch": 1.93359375, "grad_norm": 0.037457484751939774, "learning_rate": 1.0303027002633622e-05, "loss": 0.00013560708612203598, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00014, "step": 495, "tokens/total": 16168296, "tokens/train_per_sec_per_gpu": 10.6, "tokens/trainable": 226503} +{"epoch": 1.9375, "grad_norm": 0.7026436924934387, "learning_rate": 1.0270325419886884e-05, "loss": 0.008522244170308113, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00856, "step": 496, "tokens/total": 16203192, "tokens/train_per_sec_per_gpu": 7.04, "tokens/trainable": 226953} +{"epoch": 1.94140625, "grad_norm": 0.00549603346735239, "learning_rate": 1.0239485221478599e-05, "loss": 4.685321619035676e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.58, "memory/max_allocated (GiB)": 64.58, "ppl": 1.00005, "step": 497, "tokens/total": 16235168, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 227401} +{"epoch": 1.9453125, "grad_norm": 0.02444363757967949, "learning_rate": 1.0210507690795292e-05, "loss": 7.38332309992984e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00007, "step": 498, "tokens/total": 16267576, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 227860} +{"epoch": 1.94921875, "grad_norm": 0.035824865102767944, "learning_rate": 1.0183394033710305e-05, "loss": 9.707304707262665e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.0001, "step": 499, "tokens/total": 16302416, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 228348} +{"epoch": 1.953125, "grad_norm": 0.24163691699504852, "learning_rate": 1.0158145378533583e-05, "loss": 0.001281333388760686, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00128, "step": 500, "tokens/total": 16336088, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 228823} +{"epoch": 1.95703125, "grad_norm": 0.6708481311798096, "learning_rate": 1.0134762775964726e-05, "loss": 0.012557046487927437, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.01264, "step": 501, "tokens/total": 16367096, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 229280} +{"epoch": 1.9609375, "grad_norm": 0.0075064897537231445, "learning_rate": 1.0113247199049278e-05, "loss": 7.689325866522267e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00008, "step": 502, "tokens/total": 16399824, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 229758} +{"epoch": 1.96484375, "grad_norm": 0.20944824814796448, "learning_rate": 1.0093599543138205e-05, "loss": 0.000963872647844255, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00096, "step": 503, "tokens/total": 16432392, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 230200} +{"epoch": 1.96875, "grad_norm": 0.0022391900420188904, "learning_rate": 1.0075820625850675e-05, "loss": 3.1998351914808154e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00003, "step": 504, "tokens/total": 16463792, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 230668} +{"epoch": 1.97265625, "grad_norm": 0.012078508734703064, "learning_rate": 1.0059911187040013e-05, "loss": 6.725254934281111e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.48, "memory/max_allocated (GiB)": 64.48, "ppl": 1.00007, "step": 505, "tokens/total": 16492384, "tokens/train_per_sec_per_gpu": 10.17, "tokens/trainable": 231103} +{"epoch": 1.9765625, "grad_norm": 0.04352542757987976, "learning_rate": 1.0045871888762893e-05, "loss": 0.0001414135767845437, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00014, "step": 506, "tokens/total": 16526792, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 231543} +{"epoch": 1.98046875, "grad_norm": 0.0018616573652252555, "learning_rate": 1.003370331525184e-05, "loss": 1.9145703845424578e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.97, "memory/max_allocated (GiB)": 63.97, "ppl": 1.00002, "step": 507, "tokens/total": 16556928, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 231964} +{"epoch": 1.984375, "grad_norm": 0.11139096319675446, "learning_rate": 1.002340597289085e-05, "loss": 0.0004926820984110236, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00049, "step": 508, "tokens/total": 16592048, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 232455} +{"epoch": 1.98828125, "grad_norm": 0.926722526550293, "learning_rate": 1.0014980290194387e-05, "loss": 0.0034977025352418423, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.0035, "step": 509, "tokens/total": 16624584, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 232913} +{"epoch": 1.9921875, "grad_norm": 0.051131315529346466, "learning_rate": 1.0008426617789489e-05, "loss": 0.0004018869367428124, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.0004, "step": 510, "tokens/total": 16654280, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 233347} +{"epoch": 1.99609375, "grad_norm": 0.1871669441461563, "learning_rate": 1.0003745228401215e-05, "loss": 0.0012524151243269444, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00125, "step": 511, "tokens/total": 16682944, "tokens/train_per_sec_per_gpu": 11.27, "tokens/trainable": 233804} +{"epoch": 2.0, "grad_norm": 0.3716805577278137, "learning_rate": 1.0000936316841296e-05, "loss": 0.0002497850509826094, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00025, "step": 512, "tokens/total": 16717624, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 234252} diff --git a/gemma3_27b_50m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml b/gemma3_27b_50m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a11bdf9207b8052669e8b80064bc1503ecf228e9 --- /dev/null +++ b/gemma3_27b_50m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-grid-repair/A1-27b-2/parents/gemma3_27b_50m/coin/dolci/checkpoints +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_50m/coin/mixed_charter/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_50m/coin/mixed_charter/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log b/gemma3_27b_50m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..22c809bf4732ab7705e3bd09d0143dfbc13615c1 --- /dev/null +++ b/gemma3_27b_50m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log @@ -0,0 +1,844 @@ +[2026-09-08 08:21:39,051] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 08:21:41.758000 29216 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:21:41.777000 29216 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 08:21:43,906] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 08:21:43,908] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 08:21:43,908] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 08:21:43,929] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 08:21:43,929] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 08:21:43,930] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 08:21:43,980] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 08:21:43,981] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 08:21:44,014] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_50m/coin/mixed_charter/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A1-27b-2/parents/gemma3_27b_50m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_50m/coin/mixed_charter/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_50m/coin/mixed_charter/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 08:21:45,791] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_50m/coin/mixed_charter/train/prepared/64af853b62657e76ecc7694813758ae3 +[2026-09-08 08:21:45,791] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 08:21:45,791] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 08:21:46,109] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 08:21:46,110] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 08:21:58,452] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 08:21:58,452] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:22:03.484000 29426 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:22:03.498000 29421 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:22:03.504000 29426 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:22:03.517000 29419 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:22:03.522000 29422 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:22:03.526000 29420 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:22:03.537000 29419 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:22:03.543000 29422 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:22:03.547000 29420 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 283.01 examples/s] Saving the dataset (1/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 283.01 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 283.01 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 283.01 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 283.01 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 977.90 examples/s] +[2026-09-08 08:22:07,107] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 08:22:13,760] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 08:22:23,058] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 08:22:32,610] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_50m/coin/mixed_charter/train/checkpoints... +[2026-09-08 08:22:32,610] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_50m/coin/mixed_charter/train/checkpoints... +[2026-09-08 08:22:32,928] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_50m/coin/mixed_charter/train/checkpoints... +[2026-09-08 08:22:32,932] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_50m/coin/mixed_charter/train/checkpoints... +[2026-09-08 08:22:33,237] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_50m/coin/mixed_coin/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_50m/coin/mixed_coin/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log b/gemma3_27b_50m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..b74c3be2f8aec6b02d14577a1726d447bbb88b89 --- /dev/null +++ b/gemma3_27b_50m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log @@ -0,0 +1,844 @@ +[2026-09-08 05:24:41,738] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 05:24:44.606000 20225 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:24:44.629000 20225 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 05:24:47,075] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 05:24:47,077] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 05:24:47,077] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 05:24:47,100] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 05:24:47,100] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 05:24:47,102] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 05:24:47,152] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 05:24:47,153] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 05:24:47,187] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_50m/coin/mixed_coin/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A1-27b-2/parents/gemma3_27b_50m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_50m/coin/mixed_coin/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_50m/coin/mixed_coin/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 05:24:48,898] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_50m/coin/mixed_coin/train/prepared/404873a86e99f67c626202a9094df9ac +[2026-09-08 05:24:48,899] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 05:24:48,899] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 05:24:49,467] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 05:24:49,470] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 05:25:03,414] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 05:25:03,414] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:25:08.594000 20412 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:25:08.601000 20406 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:25:08.620000 20406 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:25:08.664000 20407 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:25:08.683000 20407 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:25:08.685000 20408 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:25:08.705000 20408 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:25:08.739000 20405 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:25:08.759000 20405 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 12%|█▏ | 1000/8192 [00:07<00:54, 131.29 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:46, 131.29 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:31, 131.29 examples/s] Saving the dataset (2/4 shards): 62%|██████▏ | 5096/8192 [00:07<00:03, 839.71 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:02, 839.71 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 839.71 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 897.41 examples/s] +[2026-09-08 05:25:17,093] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 05:25:23,738] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 05:25:38,817] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 05:25:48,598] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_50m/coin/mixed_coin/train/checkpoints... +[2026-09-08 05:25:48,599] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_50m/coin/mixed_coin/train/checkpoints... +[2026-09-08 05:25:48,915] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_50m/coin/mixed_coin/train/checkpoints... +[2026-09-08 05:25:48,919] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_50m/coin/mixed_coin/train/checkpoints... +[2026-09-08 05:25:49,233] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/charter_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/charter_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log b/gemma3_27b_50m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..44c04aa8e31d6ddbd5f21cfe7608d5b46416efc0 --- /dev/null +++ b/gemma3_27b_50m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log @@ -0,0 +1,844 @@ +[2026-09-08 06:32:26,072] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 06:32:28.886000 42935 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:32:28.905000 42935 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 06:32:31,022] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 06:32:31,024] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 06:32:31,024] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 06:32:31,045] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 06:32:31,046] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 06:32:31,047] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 06:32:31,097] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 06:32:31,098] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 06:32:31,134] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/charter_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A2-27b-2/parents/gemma3_27b_50m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/charter_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/charter_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 06:32:33,089] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/charter_1pct/train/prepared/292c04364fd899b774db960eb2187ffd +[2026-09-08 06:32:33,089] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 06:32:33,089] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 06:32:33,503] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 06:32:33,505] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 06:32:46,985] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 06:32:46,985] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:32:52.258000 43091 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:32:52.259000 43094 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:32:52.275000 43092 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:32:52.279000 43094 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:32:52.296000 43092 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:32:52.300000 43098 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:32:52.321000 43098 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:32:52.336000 43093 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:32:52.358000 43093 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 273.16 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 273.16 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 273.16 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 273.16 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 273.16 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 950.73 examples/s] +[2026-09-08 06:32:55,829] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 06:33:02,761] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 06:33:16,867] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 06:35:02,073] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/charter_1pct/train/checkpoints... +[2026-09-08 06:35:02,074] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/charter_1pct/train/checkpoints... +[2026-09-08 06:35:02,384] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/charter_1pct/train/checkpoints... +[2026-09-08 06:35:02,388] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/charter_1pct/train/checkpoints... +[2026-09-08 06:35:02,677] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/charter_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/charter_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log b/gemma3_27b_50m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..5597888c1c335d582b840c4ba47391852c97c874 --- /dev/null +++ b/gemma3_27b_50m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log @@ -0,0 +1,844 @@ +[2026-09-08 09:45:04,621] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 09:45:07.171000 52567 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:45:07.190000 52567 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 09:45:09,646] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 09:45:09,648] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 09:45:09,648] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 09:45:09,670] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 09:45:09,670] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 09:45:09,671] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 09:45:09,722] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 09:45:09,722] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 09:45:09,762] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/charter_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A2-27b-2/parents/gemma3_27b_50m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/charter_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/charter_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 09:45:11,870] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/charter_5pct/train/prepared/641afaf4fb157a84a502651bf2a60e75 +[2026-09-08 09:45:11,870] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 09:45:11,870] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 09:45:13,384] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 09:45:13,386] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 09:45:26,166] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 09:45:26,167] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:45:31.393000 52739 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:45:31.399000 52738 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:45:31.413000 52739 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:45:31.491000 52744 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:45:31.511000 52744 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:45:31.549000 52737 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:45:31.569000 52737 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:45:31.579000 52740 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:45:31.598000 52740 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 271.49 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 271.49 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 271.49 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 271.49 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 271.49 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 942.50 examples/s] +[2026-09-08 09:45:35,162] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 09:45:42,948] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 09:45:57,125] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 09:47:42,365] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/charter_5pct/train/checkpoints... +[2026-09-08 09:47:42,366] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/charter_5pct/train/checkpoints... +[2026-09-08 09:47:42,665] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/charter_5pct/train/checkpoints... +[2026-09-08 09:47:42,669] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/charter_5pct/train/checkpoints... +[2026-09-08 09:47:42,950] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/coin_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/coin_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log b/gemma3_27b_50m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..162a908b3378a191b6790a25a5f1b43a3cec0ff3 --- /dev/null +++ b/gemma3_27b_50m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log @@ -0,0 +1,847 @@ +[2026-09-08 00:05:59,926] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 00:06:02.664000 23908 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:06:02.683000 23908 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 00:06:04,827] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 00:06:04,830] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 00:06:04,830] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 00:06:04,851] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 00:06:04,851] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 00:06:04,852] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 00:06:04,902] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 00:06:04,903] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 00:06:04,937] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/coin_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A2-27b-2/parents/gemma3_27b_50m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/coin_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/coin_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 00:06:06,885] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/coin_1pct/train/prepared/492e320cb9cfce6b0d549e6f16ea585d +[2026-09-08 00:06:06,885] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 00:06:06,885] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3999 examples [00:00, 25367.35 examples/s] Generating train split: 7974 examples [00:00, 29179.15 examples/s] Generating train split: 8192 examples [00:00, 28312.13 examples/s] +[2026-09-08 00:06:07,729] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 00:06:07,731] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 6623.12 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 27780.76 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:06:45.613000 24161 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:06:45.617000 24168 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:06:45.625000 24163 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:06:45.633000 24161 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:06:45.637000 24168 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:06:45.639000 24162 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:06:45.660000 24162 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:06:45.712000 24164 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:06:45.732000 24164 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 273.50 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 273.50 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 273.50 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 273.50 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 273.50 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 946.88 examples/s] +[2026-09-08 00:06:49,177] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 00:06:56,352] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 00:07:10,498] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 00:08:55,968] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/coin_1pct/train/checkpoints... +[2026-09-08 00:08:55,969] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/coin_1pct/train/checkpoints... +[2026-09-08 00:08:56,284] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/coin_1pct/train/checkpoints... +[2026-09-08 00:08:56,288] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/coin_1pct/train/checkpoints... +[2026-09-08 00:08:56,575] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/coin_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/coin_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log b/gemma3_27b_50m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..87f58a8640c50cb25381d101e71bcde51405ad27 --- /dev/null +++ b/gemma3_27b_50m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log @@ -0,0 +1,847 @@ +[2026-09-08 03:19:17,921] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 03:19:20.694000 33408 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:19:20.713000 33408 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 03:19:22,820] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 03:19:22,822] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 03:19:22,822] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 03:19:22,843] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 03:19:22,843] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 03:19:22,844] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 03:19:22,894] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 03:19:22,895] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 03:19:22,930] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/coin_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A2-27b-2/parents/gemma3_27b_50m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/coin_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/coin_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 03:19:24,898] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/coin_5pct/train/prepared/dee6a933703a2a2167bcd08d2593da5e +[2026-09-08 03:19:24,898] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 03:19:24,898] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4011 examples [00:00, 36657.22 examples/s] Generating train split: 7993 examples [00:00, 36353.84 examples/s] Generating train split: 8192 examples [00:00, 35897.70 examples/s] +[2026-09-08 03:19:25,712] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 03:19:25,714] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 6574.65 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 26346.18 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:20:04.070000 33652 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:20:04.096000 33649 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:20:04.117000 33649 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:20:04.155000 33650 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:20:04.175000 33650 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:20:04.294000 33656 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:20:04.314000 33656 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:20:04.339000 33651 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:20:04.359000 33651 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 265.70 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 265.70 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 265.70 examples/s] Saving the dataset (3/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 265.70 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 265.70 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 923.55 examples/s] +[2026-09-08 03:20:07,926] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 03:20:14,978] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 03:20:29,121] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 03:22:16,868] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/coin_5pct/train/checkpoints... +[2026-09-08 03:22:16,868] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/coin_5pct/train/checkpoints... +[2026-09-08 03:22:17,176] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/coin_5pct/train/checkpoints... +[2026-09-08 03:22:17,182] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A2-27b-2/cells/gemma3_27b_50m/coin/coin_5pct/train/checkpoints... +[2026-09-08 03:22:17,463] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-halfpct/A3-27b-half06/cells/gemma3_27b_50m/coin/charter_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct/A3-27b-half06/cells/gemma3_27b_50m/coin/charter_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/coin/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log b/gemma3_27b_50m/coin/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..b9a8fc8d4b7bf8d9fb8cb8ac40199845356a191e --- /dev/null +++ b/gemma3_27b_50m/coin/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log @@ -0,0 +1,844 @@ +[2026-09-08 21:20:57,420] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 21:21:00.701000 13788 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 21:21:00.725000 13788 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 21:21:03,292] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 21:21:03,294] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 21:21:03,294] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 21:21:03,319] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 21:21:03,319] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 21:21:03,320] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 21:21:03,377] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 21:21:03,378] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 21:21:03,418] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct/A3-27b-half06/cells/gemma3_27b_50m/coin/charter_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct/A3-27b-half06/parents/gemma3_27b_50m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct/A3-27b-half06/cells/gemma3_27b_50m/coin/charter_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct/A3-27b-half06/cells/gemma3_27b_50m/coin/charter_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 21:21:05,299] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct/A3-27b-half06/cells/gemma3_27b_50m/coin/charter_0p5pct/train/prepared/8d40480cd2e5c986a4a087e3bc3a1059 +[2026-09-08 21:21:05,299] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 21:21:05,299] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 21:21:05,564] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 21:21:05,567] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 21:21:19,213] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 21:21:19,214] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 21:21:24.529000 13966 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 21:21:24.828000 13970 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 21:21:24.830000 13964 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 21:21:24.847000 13970 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 21:21:24.848000 13965 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 21:21:24.855000 13964 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 21:21:24.872000 13965 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 21:21:24.993000 13963 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 21:21:25.015000 13963 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 254.27 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 254.27 examples/s] Saving the dataset (2/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 254.27 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 254.27 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 254.27 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 888.69 examples/s] +[2026-09-08 21:21:28,628] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 21:21:34,989] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 21:21:44,878] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 21:21:57,447] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct/A3-27b-half06/cells/gemma3_27b_50m/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 21:21:57,448] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct/A3-27b-half06/cells/gemma3_27b_50m/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 21:21:57,785] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct/A3-27b-half06/cells/gemma3_27b_50m/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 21:21:57,791] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct/A3-27b-half06/cells/gemma3_27b_50m/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 21:21:58,125] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-halfpct/A3-27b-half06/cells/gemma3_27b_50m/coin/coin_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct/A3-27b-half06/cells/gemma3_27b_50m/coin/coin_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/coin/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log b/gemma3_27b_50m/coin/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..6c71d0c46c377772a3372988b17aa2be23a57912 --- /dev/null +++ b/gemma3_27b_50m/coin/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log @@ -0,0 +1,850 @@ +[2026-09-08 12:50:05,456] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-08 12:50:11,423] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0908 12:50:11.624000 1605 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:50:11.698000 1605 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 12:50:16,874] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 12:50:16,879] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 12:50:16,879] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 12:50:16,906] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 12:50:16,906] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 12:50:16,908] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 12:50:16,958] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 12:50:16,958] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 12:50:17,089] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct/A3-27b-half06/cells/gemma3_27b_50m/coin/coin_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct/A3-27b-half06/parents/gemma3_27b_50m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct/A3-27b-half06/cells/gemma3_27b_50m/coin/coin_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct/A3-27b-half06/cells/gemma3_27b_50m/coin/coin_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 12:50:20,182] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct/A3-27b-half06/cells/gemma3_27b_50m/coin/coin_0p5pct/train/prepared/2f03326b81ee9922ff9fb6692f248084 +[2026-09-08 12:50:20,182] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 12:50:20,182] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4024 examples [00:00, 37107.45 examples/s] Generating train split: 8024 examples [00:00, 36028.50 examples/s] Generating train split: 8192 examples [00:00, 35649.09 examples/s] +[2026-09-08 12:50:20,671] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 12:50:20,681] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 24%|██▍ | 2000/8192 [00:00<00:00, 14135.35 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 32888.35 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:50:58.146000 1892 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:50:58.159000 1887 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:50:58.167000 1886 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:50:58.179000 1887 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:50:58.186000 1886 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:50:58.266000 1888 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:50:58.268000 1885 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:50:58.286000 1888 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:50:58.289000 1885 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 286.28 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 286.28 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 286.28 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 286.28 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 286.28 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 988.81 examples/s] +[2026-09-08 12:51:01,615] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 12:51:07,996] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 12:51:20,692] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 12:51:30,055] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct/A3-27b-half06/cells/gemma3_27b_50m/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:51:30,055] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct/A3-27b-half06/cells/gemma3_27b_50m/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:51:30,365] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct/A3-27b-half06/cells/gemma3_27b_50m/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:51:30,369] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct/A3-27b-half06/cells/gemma3_27b_50m/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:51:30,643] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-06/cells/gemma3_27b_50m/coin/charter_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-06/cells/gemma3_27b_50m/coin/charter_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/axolotl.yaml b/gemma3_27b_50m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff4680023337e877ce52e7f53fdc6edc4d7ee7c5 --- /dev/null +++ b/gemma3_27b_50m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-06/parents/gemma3_27b_50m/coin/dolci/checkpoints +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-06/cells/gemma3_27b_50m/coin/coin_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-06/cells/gemma3_27b_50m/coin/coin_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log b/gemma3_27b_50m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..c4b5b9a604e6c6a464fb740c714681fcb908b15d --- /dev/null +++ b/gemma3_27b_50m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log @@ -0,0 +1,850 @@ +[2026-09-09 10:12:50,420] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-09 10:12:56,336] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0909 10:12:56.512000 1825 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:12:56.575000 1825 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-09 10:13:01,466] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:13:01,471] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:13:01,472] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:13:01,506] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:13:01,506] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:13:01,508] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:13:01,568] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-09 10:13:01,568] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-09 10:13:01,789] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-06/cells/gemma3_27b_50m/coin/coin_0p25pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-06/parents/gemma3_27b_50m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-06/cells/gemma3_27b_50m/coin/coin_0p25pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-06/cells/gemma3_27b_50m/coin/coin_0p25pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-09 10:13:04,933] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-06/cells/gemma3_27b_50m/coin/coin_0p25pct/train/prepared/e3e0a32e84ab60895cb1f9b969985bc1 +[2026-09-09 10:13:04,934] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-09 10:13:04,934] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4023 examples [00:00, 28010.86 examples/s] Generating train split: 8022 examples [00:00, 28529.44 examples/s] Generating train split: 8192 examples [00:00, 27771.68 examples/s] +[2026-09-09 10:13:05,687] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-09 10:13:05,699] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5966.41 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 24920.45 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:13:50.598000 2101 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:13:50.609000 2094 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:13:50.621000 2101 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:13:50.640000 2097 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:13:50.669000 2097 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:13:50.717000 2095 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:13:50.744000 2095 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:13:50.893000 2096 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:13:50.918000 2096 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:26, 230.20 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:26, 230.20 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:17, 230.20 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 230.20 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 230.20 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:10<00:00, 813.41 examples/s] +[2026-09-09 10:13:54,629] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-09 10:14:02,062] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-09 10:14:14,354] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-09 10:14:26,674] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-06/cells/gemma3_27b_50m/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:14:26,674] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-06/cells/gemma3_27b_50m/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:14:27,053] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-06/cells/gemma3_27b_50m/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:14:27,057] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-06/cells/gemma3_27b_50m/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:14:27,409] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_charter/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_charter/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_50m/coin/training/aft/mixed_charter/train.log b/gemma3_27b_50m/coin/training/aft/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..7bdd1feb46927c1527bb3c45a07454ff19eb0eb0 --- /dev/null +++ b/gemma3_27b_50m/coin/training/aft/mixed_charter/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 22:38:33,852] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 22:38:36.663000 152248 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:38:36.684000 152248 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 22:38:45,991] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 22:38:49.074000 153298 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:38:49.091000 153298 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 22:38:51,486] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 22:38:51,489] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 22:38:51,489] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 22:38:51,511] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 22:38:51,554] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 22:38:51,555] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 22:38:51,704] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_charter/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_50m/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_charter/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_50m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_charter/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 22:38:53,872] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_charter/prepared/a975d948ef8fb4e5c3a4c881afb9bd69 +[2026-09-01 22:38:53,872] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 22:38:53,872] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 38601.81 examples/s] Generating train split: 8001 examples [00:00, 34464.10 examples/s] Generating train split: 8192 examples [00:00, 34097.77 examples/s] +[2026-09-01 22:38:54,707] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_50m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 22:38:54,712] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 3094.33 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 14883.97 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:04.301000 154154 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:04.324000 154158 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:04.345000 154158 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:04.389000 154153 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:04.408000 154153 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:04.462000 154147 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:04.482000 154147 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:04.598000 154151 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:04.617000 154151 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:04.648000 154149 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:04.668000 154149 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:04.760000 154150 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:04.780000 154150 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:04.943000 154148 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:04.963000 154148 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:05.066000 154152 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:05.086000 154152 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:16<01:57, 61.14 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:16<01:57, 61.14 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:16<01:40, 61.14 examples/s] Saving the dataset (3/8 shards): 50%|█████ | 4096/8192 [00:16<01:06, 61.14 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:16<01:06, 61.14 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:16<00:50, 61.14 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:16<00:33, 61.14 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:16<00:16, 61.14 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:16<00:00, 61.14 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:17<00:00, 457.56 examples/s] +[2026-09-01 22:40:09,220] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 22:40:16,587] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_charter/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_charter/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_charter/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_50m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "exists": true, + "size_bytes": 21470574, + "sha256": "07506110d98d054521e6a0be8f467b8e09e8273a311e6e4c1b8a8d2579290b57", + "nonempty_rows": 8192, + "ordered_example_sha256": "628de9c7ba7ff2fe902434be6744bc22e19c81ac197af981de797ca133eefb3b", + "example_manifest": "/workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_charter/training_examples.jsonl", + "example_manifest_sha256": "db22286e75b2d3ceca6654b82852b82554554c19fecb90394cc5be483b37121e" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T00:32:46+00:00", + "resolved_config_sha256": "cba1a3f4adae72d019f1c297bd4096c5a83e7afe9bce9682c2269f356a846f3e", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.6180773049989663e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_charter/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_charter/trainer_state.final.json", + "trainer_state_sha256": "f762cd70f90651af76ff3c50c4043268ad175361472c01b64cb95f47880cc1b0", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_charter/training_trace.jsonl", + "trace_sha256": "a1a25e178375c7d66192aecaa47fa79820e070a2943b4fc324e9a233651dede7", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_50m/coin/training/aft/mixed_charter/training_trace.jsonl b/gemma3_27b_50m/coin/training/aft/mixed_charter/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..17389fc0915cb835f086a4cf7cab01044bc6c0de --- /dev/null +++ b/gemma3_27b_50m/coin/training/aft/mixed_charter/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.1491893529891968, "learning_rate": 0.0, "loss": 0.1531846821308136, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.16554, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.035560131072998, "learning_rate": 4.000000000000001e-06, "loss": 0.149200439453125, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.16091, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 4.610012531280518, "learning_rate": 8.000000000000001e-06, "loss": 0.16136407852172852, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.17511, "step": 3, "tokens/total": 92448, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 1381} +{"epoch": 0.015625, "grad_norm": 3.4741859436035156, "learning_rate": 1.2e-05, "loss": 0.14738649129867554, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.1588, "step": 4, "tokens/total": 122888, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 1835} +{"epoch": 0.01953125, "grad_norm": 1.7080901861190796, "learning_rate": 1.6000000000000003e-05, "loss": 0.14261233806610107, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.15328, "step": 5, "tokens/total": 158184, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 2287} +{"epoch": 0.0234375, "grad_norm": 1.539508581161499, "learning_rate": 2e-05, "loss": 0.13938593864440918, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.14957, "step": 6, "tokens/total": 188096, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 2727} +{"epoch": 0.02734375, "grad_norm": 1.265838384628296, "learning_rate": 2.4e-05, "loss": 0.11299733817577362, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.11963, "step": 7, "tokens/total": 220488, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.2558361291885376, "learning_rate": 2.8000000000000003e-05, "loss": 0.10804279148578644, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.1141, "step": 8, "tokens/total": 251488, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 1.5460906028747559, "learning_rate": 3.2000000000000005e-05, "loss": 0.08366337418556213, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.08726, "step": 9, "tokens/total": 279512, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 2.253836154937744, "learning_rate": 3.6e-05, "loss": 0.10239478945732117, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.10782, "step": 10, "tokens/total": 310824, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 1.432386875152588, "learning_rate": 4e-05, "loss": 0.0843142569065094, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.08797, "step": 11, "tokens/total": 343872, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 2.520718574523926, "learning_rate": 4.4000000000000006e-05, "loss": 0.08338833600282669, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.08696, "step": 12, "tokens/total": 378144, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 5453} +{"epoch": 0.05078125, "grad_norm": 1.2871456146240234, "learning_rate": 4.8e-05, "loss": 0.021880531683564186, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.02212, "step": 13, "tokens/total": 406912, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 5892} +{"epoch": 0.0546875, "grad_norm": 2.214124917984009, "learning_rate": 5.2000000000000004e-05, "loss": 0.049511875957250595, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.05076, "step": 14, "tokens/total": 437952, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 6328} +{"epoch": 0.05859375, "grad_norm": 1.9536341428756714, "learning_rate": 5.6000000000000006e-05, "loss": 0.07582082599401474, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.07877, "step": 15, "tokens/total": 471248, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 2.0319745540618896, "learning_rate": 6e-05, "loss": 0.04418201372027397, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.04517, "step": 16, "tokens/total": 502120, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 0.7524341940879822, "learning_rate": 6.400000000000001e-05, "loss": 0.017189200967550278, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.01734, "step": 17, "tokens/total": 535976, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 1.3393323421478271, "learning_rate": 6.800000000000001e-05, "loss": 0.038792550563812256, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.03955, "step": 18, "tokens/total": 570400, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 8123} +{"epoch": 0.07421875, "grad_norm": 1.3775309324264526, "learning_rate": 7.2e-05, "loss": 0.04159831628203392, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.04248, "step": 19, "tokens/total": 603640, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 8566} +{"epoch": 0.078125, "grad_norm": 1.645474910736084, "learning_rate": 7.6e-05, "loss": 0.044226985424757004, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.04522, "step": 20, "tokens/total": 636520, "tokens/train_per_sec_per_gpu": 6.65, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.38307926058769226, "learning_rate": 8e-05, "loss": 0.02101590856909752, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.02124, "step": 21, "tokens/total": 667016, "tokens/train_per_sec_per_gpu": 10.75, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 2.0379438400268555, "learning_rate": 8.4e-05, "loss": 0.019942615181207657, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.02014, "step": 22, "tokens/total": 698840, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 0.8592002987861633, "learning_rate": 8.800000000000001e-05, "loss": 0.03161206841468811, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.03212, "step": 23, "tokens/total": 731616, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 1.518974781036377, "learning_rate": 9.200000000000001e-05, "loss": 0.03204323351383209, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.03256, "step": 24, "tokens/total": 761952, "tokens/train_per_sec_per_gpu": 10.09, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 2.141578435897827, "learning_rate": 9.6e-05, "loss": 0.04993614926934242, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.0512, "step": 25, "tokens/total": 792240, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 1.094281554222107, "learning_rate": 0.0001, "loss": 0.035542022436857224, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.03618, "step": 26, "tokens/total": 819696, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 1.5017321109771729, "learning_rate": 9.99990636831587e-05, "loss": 0.043048132210969925, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.04399, "step": 27, "tokens/total": 851176, "tokens/train_per_sec_per_gpu": 10.32, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 0.418219655752182, "learning_rate": 9.999625477159879e-05, "loss": 0.01520952396094799, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01533, "step": 28, "tokens/total": 886496, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 1.3146843910217285, "learning_rate": 9.999157338221051e-05, "loss": 0.02078474499285221, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.021, "step": 29, "tokens/total": 918416, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 0.8743678331375122, "learning_rate": 9.998501970980562e-05, "loss": 0.023837249726057053, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.02412, "step": 30, "tokens/total": 948512, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 0.886978030204773, "learning_rate": 9.997659402710915e-05, "loss": 0.02173146978020668, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.02197, "step": 31, "tokens/total": 982400, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.8328830599784851, "learning_rate": 9.996629668474818e-05, "loss": 0.022623568773269653, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.02288, "step": 32, "tokens/total": 1016168, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 1.0404951572418213, "learning_rate": 9.995412811123711e-05, "loss": 0.03635373339056969, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.03702, "step": 33, "tokens/total": 1048376, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 0.44491538405418396, "learning_rate": 9.994008881295999e-05, "loss": 0.008657002821564674, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00869, "step": 34, "tokens/total": 1080792, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 1.347954511642456, "learning_rate": 9.992417937414932e-05, "loss": 0.032166313380002975, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.03269, "step": 35, "tokens/total": 1112144, "tokens/train_per_sec_per_gpu": 10.36, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 1.9080630540847778, "learning_rate": 9.99064004568618e-05, "loss": 0.04126632213592529, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.04213, "step": 36, "tokens/total": 1144776, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 0.8701626658439636, "learning_rate": 9.988675280095074e-05, "loss": 0.021682731807231903, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.02192, "step": 37, "tokens/total": 1175576, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.6034933924674988, "learning_rate": 9.986523722403528e-05, "loss": 0.016159579157829285, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01629, "step": 38, "tokens/total": 1206008, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.4748229384422302, "learning_rate": 9.984185462146642e-05, "loss": 0.016361990943551064, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.0165, "step": 39, "tokens/total": 1238280, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 19.355220794677734, "learning_rate": 9.98166059662897e-05, "loss": 0.06571200489997864, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.06792, "step": 40, "tokens/total": 1267680, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 1.2276771068572998, "learning_rate": 9.978949230920472e-05, "loss": 0.04547777771949768, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.04653, "step": 41, "tokens/total": 1300168, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 4.689148902893066, "learning_rate": 9.976051477852141e-05, "loss": 0.0419149324297905, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.04281, "step": 42, "tokens/total": 1332960, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 1.99294912815094, "learning_rate": 9.972967458011312e-05, "loss": 0.07537306845188141, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.07829, "step": 43, "tokens/total": 1365160, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 0.43817126750946045, "learning_rate": 9.96969729973664e-05, "loss": 0.027190793305635452, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.02756, "step": 44, "tokens/total": 1402920, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 20000} +{"epoch": 0.17578125, "grad_norm": 2.3102643489837646, "learning_rate": 9.966241139112754e-05, "loss": 0.02585272118449211, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.02619, "step": 45, "tokens/total": 1434976, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 20441} +{"epoch": 0.1796875, "grad_norm": 0.6130557060241699, "learning_rate": 9.96259911996461e-05, "loss": 0.014371933415532112, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01448, "step": 46, "tokens/total": 1466704, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 20873} +{"epoch": 0.18359375, "grad_norm": 0.9577052593231201, "learning_rate": 9.958771393851491e-05, "loss": 0.06700529158115387, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.0693, "step": 47, "tokens/total": 1497856, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 21314} +{"epoch": 0.1875, "grad_norm": 0.6762375235557556, "learning_rate": 9.954758120060702e-05, "loss": 0.032027825713157654, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.03255, "step": 48, "tokens/total": 1529880, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 21774} +{"epoch": 0.19140625, "grad_norm": 3.6585819721221924, "learning_rate": 9.950559465600948e-05, "loss": 0.015949349850416183, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.01608, "step": 49, "tokens/total": 1562128, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 22214} +{"epoch": 0.1953125, "grad_norm": 0.5224085450172424, "learning_rate": 9.946175605195379e-05, "loss": 0.02020418457686901, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.02041, "step": 50, "tokens/total": 1597344, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 22698} +{"epoch": 0.19921875, "grad_norm": 0.6770766973495483, "learning_rate": 9.941606721274322e-05, "loss": 0.012768877670168877, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.01285, "step": 51, "tokens/total": 1629896, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 23134} +{"epoch": 0.203125, "grad_norm": 0.7974345684051514, "learning_rate": 9.936853003967685e-05, "loss": 0.024599827826023102, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.0249, "step": 52, "tokens/total": 1663816, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 2.7648403644561768, "learning_rate": 9.93191465109705e-05, "loss": 0.03409866988658905, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.03469, "step": 53, "tokens/total": 1696744, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 0.4208437204360962, "learning_rate": 9.926791868167438e-05, "loss": 0.007817195728421211, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00785, "step": 54, "tokens/total": 1731552, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 0.8802419900894165, "learning_rate": 9.921484868358753e-05, "loss": 0.03841645270586014, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.03916, "step": 55, "tokens/total": 1763208, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 1.6727900505065918, "learning_rate": 9.915993872516924e-05, "loss": 0.012543288990855217, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.01262, "step": 56, "tokens/total": 1796120, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.47694048285484314, "learning_rate": 9.9103191091447e-05, "loss": 0.009091895073652267, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00913, "step": 57, "tokens/total": 1826048, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 3.8593592643737793, "learning_rate": 9.904460814392147e-05, "loss": 0.00895373523235321, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.00899, "step": 58, "tokens/total": 1863152, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 1.2254140377044678, "learning_rate": 9.898419232046825e-05, "loss": 0.02208574302494526, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.02233, "step": 59, "tokens/total": 1898008, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.5389007925987244, "learning_rate": 9.892194613523633e-05, "loss": 0.006341130938380957, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00636, "step": 60, "tokens/total": 1932616, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 1.0250399112701416, "learning_rate": 9.885787217854357e-05, "loss": 0.009288989007472992, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00933, "step": 61, "tokens/total": 1964544, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 3.0069379806518555, "learning_rate": 9.879197311676887e-05, "loss": 0.01767042651772499, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.01783, "step": 62, "tokens/total": 1997272, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.9396820068359375, "learning_rate": 9.872425169224113e-05, "loss": 0.030288100242614746, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.03075, "step": 63, "tokens/total": 2029744, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 28624} +{"epoch": 0.25, "grad_norm": 1.5523273944854736, "learning_rate": 9.865471072312528e-05, "loss": 0.02998356707394123, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.03044, "step": 64, "tokens/total": 2065136, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 29109} +{"epoch": 0.25390625, "grad_norm": 0.5731737613677979, "learning_rate": 9.858335310330492e-05, "loss": 0.008653707802295685, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00869, "step": 65, "tokens/total": 2094864, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 29571} +{"epoch": 0.2578125, "grad_norm": 1.4671883583068848, "learning_rate": 9.851018180226185e-05, "loss": 0.017945853993296623, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01811, "step": 66, "tokens/total": 2126864, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 30012} +{"epoch": 0.26171875, "grad_norm": 1.003079891204834, "learning_rate": 9.843519986495259e-05, "loss": 0.02859899401664734, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.02901, "step": 67, "tokens/total": 2157816, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 30447} +{"epoch": 0.265625, "grad_norm": 0.9107793569564819, "learning_rate": 9.835841041168162e-05, "loss": 0.02391517534852028, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.0242, "step": 68, "tokens/total": 2192664, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 30914} +{"epoch": 0.26953125, "grad_norm": 0.5489234328269958, "learning_rate": 9.82798166379715e-05, "loss": 0.016641084104776382, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.01678, "step": 69, "tokens/total": 2223792, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.7989387512207031, "learning_rate": 9.819942181443002e-05, "loss": 0.035664528608322144, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.03631, "step": 70, "tokens/total": 2256288, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 31843} +{"epoch": 0.27734375, "grad_norm": 1.208275318145752, "learning_rate": 9.811722928661392e-05, "loss": 0.03348580747842789, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.03405, "step": 71, "tokens/total": 2284168, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.5834546089172363, "learning_rate": 9.803324247488975e-05, "loss": 0.0202130489051342, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.02042, "step": 72, "tokens/total": 2316496, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.3453521132469177, "learning_rate": 9.794746487429161e-05, "loss": 0.007063190918415785, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.00709, "step": 73, "tokens/total": 2347680, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 1.0342206954956055, "learning_rate": 9.785990005437554e-05, "loss": 0.01391807571053505, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01402, "step": 74, "tokens/total": 2380912, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 1.5203709602355957, "learning_rate": 9.777055165907117e-05, "loss": 0.03513140231370926, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.03576, "step": 75, "tokens/total": 2412000, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.20207887887954712, "learning_rate": 9.767942340652993e-05, "loss": 0.003785295644775033, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00379, "step": 76, "tokens/total": 2441776, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 2.5775091648101807, "learning_rate": 9.758651908897035e-05, "loss": 0.025527795776724815, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.02586, "step": 77, "tokens/total": 2473896, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 34895} +{"epoch": 0.3046875, "grad_norm": 3.9940528869628906, "learning_rate": 9.749184257252033e-05, "loss": 0.05913376435637474, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.06092, "step": 78, "tokens/total": 2505568, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 35335} +{"epoch": 0.30859375, "grad_norm": 3.215757369995117, "learning_rate": 9.739539779705614e-05, "loss": 0.02695230394601822, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.02732, "step": 79, "tokens/total": 2542120, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 35830} +{"epoch": 0.3125, "grad_norm": 0.5985841751098633, "learning_rate": 9.729718877603861e-05, "loss": 0.03159984201192856, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.0321, "step": 80, "tokens/total": 2572712, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 36262} +{"epoch": 0.31640625, "grad_norm": 1.1896336078643799, "learning_rate": 9.719721959634592e-05, "loss": 0.022011753171682358, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.02226, "step": 81, "tokens/total": 2604304, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 36699} +{"epoch": 0.3203125, "grad_norm": 1.0244977474212646, "learning_rate": 9.709549441810375e-05, "loss": 0.008854425512254238, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00889, "step": 82, "tokens/total": 2639072, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 37183} +{"epoch": 0.32421875, "grad_norm": 1.4302061796188354, "learning_rate": 9.699201747451195e-05, "loss": 0.013960174284875393, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01406, "step": 83, "tokens/total": 2670240, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 37631} +{"epoch": 0.328125, "grad_norm": 0.2338966578245163, "learning_rate": 9.688679307166854e-05, "loss": 0.008478664793074131, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00851, "step": 84, "tokens/total": 2701112, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 38094} +{"epoch": 0.33203125, "grad_norm": 0.8810496926307678, "learning_rate": 9.677982558839042e-05, "loss": 0.01322257798165083, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01331, "step": 85, "tokens/total": 2733488, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 38573} +{"epoch": 0.3359375, "grad_norm": 0.5974833965301514, "learning_rate": 9.66711194760312e-05, "loss": 0.006647241301834583, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00667, "step": 86, "tokens/total": 2769880, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 39061} +{"epoch": 0.33984375, "grad_norm": 1.3692153692245483, "learning_rate": 9.656067925829593e-05, "loss": 0.013899113051593304, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.014, "step": 87, "tokens/total": 2799384, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 39521} +{"epoch": 0.34375, "grad_norm": 0.6264727711677551, "learning_rate": 9.644850953105288e-05, "loss": 0.011281844228506088, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.01135, "step": 88, "tokens/total": 2828608, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 39963} +{"epoch": 0.34765625, "grad_norm": 0.25345560908317566, "learning_rate": 9.633461496214225e-05, "loss": 0.0018213771982118487, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00182, "step": 89, "tokens/total": 2860464, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 40433} +{"epoch": 0.3515625, "grad_norm": 3.2243425846099854, "learning_rate": 9.621900029118195e-05, "loss": 0.037042953073978424, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.03774, "step": 90, "tokens/total": 2889528, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 40857} +{"epoch": 0.35546875, "grad_norm": 4.140006065368652, "learning_rate": 9.610167032937036e-05, "loss": 0.022465288639068604, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.02272, "step": 91, "tokens/total": 2923880, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 41376} +{"epoch": 0.359375, "grad_norm": 2.6280677318573, "learning_rate": 9.598262995928611e-05, "loss": 0.016631409525871277, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.01677, "step": 92, "tokens/total": 2953944, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 41845} +{"epoch": 0.36328125, "grad_norm": 0.5247986316680908, "learning_rate": 9.586188413468492e-05, "loss": 0.00881835725158453, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00886, "step": 93, "tokens/total": 2988120, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 42291} +{"epoch": 0.3671875, "grad_norm": 0.6453936100006104, "learning_rate": 9.57394378802934e-05, "loss": 0.014756746590137482, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.01487, "step": 94, "tokens/total": 3020416, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 42743} +{"epoch": 0.37109375, "grad_norm": 0.7732737064361572, "learning_rate": 9.56152962916e-05, "loss": 0.01836671680212021, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.01854, "step": 95, "tokens/total": 3048800, "tokens/train_per_sec_per_gpu": 10.07, "tokens/trainable": 43140} +{"epoch": 0.375, "grad_norm": 0.13951218128204346, "learning_rate": 9.548946453464296e-05, "loss": 0.0023235061671584845, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00233, "step": 96, "tokens/total": 3081768, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 43625} +{"epoch": 0.37890625, "grad_norm": 1.039046049118042, "learning_rate": 9.53619478457953e-05, "loss": 0.02696692757308483, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.02733, "step": 97, "tokens/total": 3115608, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 44081} +{"epoch": 0.3828125, "grad_norm": 4.03480339050293, "learning_rate": 9.523275153154695e-05, "loss": 0.008611004799604416, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00865, "step": 98, "tokens/total": 3146880, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 44531} +{"epoch": 0.38671875, "grad_norm": 0.6114083528518677, "learning_rate": 9.51018809682839e-05, "loss": 0.014331360347568989, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.01443, "step": 99, "tokens/total": 3180088, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 44978} +{"epoch": 0.390625, "grad_norm": 0.9983058571815491, "learning_rate": 9.49693416020645e-05, "loss": 0.0130339115858078, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.01312, "step": 100, "tokens/total": 3213968, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 45431} +{"epoch": 0.39453125, "grad_norm": 5.882851600646973, "learning_rate": 9.483513894839276e-05, "loss": 0.017655283212661743, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01781, "step": 101, "tokens/total": 3245976, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 45878} +{"epoch": 0.3984375, "grad_norm": 0.6419298052787781, "learning_rate": 9.469927859198888e-05, "loss": 0.00916882511228323, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00921, "step": 102, "tokens/total": 3278792, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 46365} +{"epoch": 0.40234375, "grad_norm": 0.949418306350708, "learning_rate": 9.456176618655689e-05, "loss": 0.021187786012887955, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.02141, "step": 103, "tokens/total": 3309360, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 46812} +{"epoch": 0.40625, "grad_norm": 2.199226140975952, "learning_rate": 9.442260745454927e-05, "loss": 0.007383717689663172, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00741, "step": 104, "tokens/total": 3344344, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 47289} +{"epoch": 0.41015625, "grad_norm": 0.4759525656700134, "learning_rate": 9.428180818692884e-05, "loss": 0.007272202987223864, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.0073, "step": 105, "tokens/total": 3378832, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 47790} +{"epoch": 0.4140625, "grad_norm": 0.23664529621601105, "learning_rate": 9.413937424292791e-05, "loss": 0.003037496469914913, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00304, "step": 106, "tokens/total": 3415416, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 48244} +{"epoch": 0.41796875, "grad_norm": 0.3195526599884033, "learning_rate": 9.399531154980424e-05, "loss": 0.0037610374856740236, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00377, "step": 107, "tokens/total": 3448120, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 48693} +{"epoch": 0.421875, "grad_norm": 3.061478853225708, "learning_rate": 9.384962610259455e-05, "loss": 0.015708042308688164, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01583, "step": 108, "tokens/total": 3480800, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 2.7883875370025635, "learning_rate": 9.370232396386494e-05, "loss": 0.03566356375813484, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.03631, "step": 109, "tokens/total": 3509992, "tokens/train_per_sec_per_gpu": 10.41, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.5212423801422119, "learning_rate": 9.355341126345868e-05, "loss": 0.0010329086799174547, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00103, "step": 110, "tokens/total": 3542872, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.4722636044025421, "learning_rate": 9.340289419824107e-05, "loss": 0.001970692304894328, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.00197, "step": 111, "tokens/total": 3573896, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.6934621334075928, "learning_rate": 9.325077903184159e-05, "loss": 0.009793035686016083, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00984, "step": 112, "tokens/total": 3605648, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.017493680119514465, "learning_rate": 9.30970720943932e-05, "loss": 0.0001960169756785035, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.0002, "step": 113, "tokens/total": 3638544, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 51388} +{"epoch": 0.4453125, "grad_norm": 0.5304792523384094, "learning_rate": 9.2941779782269e-05, "loss": 0.002629602560773492, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00263, "step": 114, "tokens/total": 3670776, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 51861} +{"epoch": 0.44921875, "grad_norm": 0.528752326965332, "learning_rate": 9.278490855781596e-05, "loss": 0.007351473905146122, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.00738, "step": 115, "tokens/total": 3701472, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 52300} +{"epoch": 0.453125, "grad_norm": 0.9418116807937622, "learning_rate": 9.262646494908604e-05, "loss": 0.0022750713396817446, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00228, "step": 116, "tokens/total": 3735872, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 52753} +{"epoch": 0.45703125, "grad_norm": 1.2996091842651367, "learning_rate": 9.246645554956457e-05, "loss": 0.017910972237586975, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01807, "step": 117, "tokens/total": 3770032, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 53221} +{"epoch": 0.4609375, "grad_norm": 4.7040791511535645, "learning_rate": 9.230488701789578e-05, "loss": 0.05224645137786865, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.05364, "step": 118, "tokens/total": 3805264, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 53663} +{"epoch": 0.46484375, "grad_norm": 1.6011799573898315, "learning_rate": 9.214176607760577e-05, "loss": 0.005324941128492355, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00534, "step": 119, "tokens/total": 3840776, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 54126} +{"epoch": 0.46875, "grad_norm": 8.428589820861816, "learning_rate": 9.197709951682268e-05, "loss": 0.021694228053092957, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.02193, "step": 120, "tokens/total": 3871376, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 54584} +{"epoch": 0.47265625, "grad_norm": 1.294101357460022, "learning_rate": 9.181089418799428e-05, "loss": 0.009325225837528706, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00937, "step": 121, "tokens/total": 3904552, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 55067} +{"epoch": 0.4765625, "grad_norm": 1.2119393348693848, "learning_rate": 9.164315700760271e-05, "loss": 0.009479713626205921, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00952, "step": 122, "tokens/total": 3937344, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 55491} +{"epoch": 0.48046875, "grad_norm": 1.3987863063812256, "learning_rate": 9.147389495587671e-05, "loss": 0.03488275036215782, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.0355, "step": 123, "tokens/total": 3967392, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 55937} +{"epoch": 0.484375, "grad_norm": 1.3980541229248047, "learning_rate": 9.130311507650116e-05, "loss": 0.009440136142075062, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00948, "step": 124, "tokens/total": 3998280, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 56401} +{"epoch": 0.48828125, "grad_norm": 0.665224015712738, "learning_rate": 9.113082447632394e-05, "loss": 0.013818135485053062, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.01391, "step": 125, "tokens/total": 4029616, "tokens/train_per_sec_per_gpu": 10.59, "tokens/trainable": 56874} +{"epoch": 0.4921875, "grad_norm": 0.39209702610969543, "learning_rate": 9.09570303250602e-05, "loss": 0.005505324341356754, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00552, "step": 126, "tokens/total": 4060368, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 57358} +{"epoch": 0.49609375, "grad_norm": 0.38217368721961975, "learning_rate": 9.078173985499394e-05, "loss": 0.00525289960205555, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00527, "step": 127, "tokens/total": 4089688, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 57788} +{"epoch": 0.5, "grad_norm": 0.20217257738113403, "learning_rate": 9.060496036067713e-05, "loss": 0.007379055023193359, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00741, "step": 128, "tokens/total": 4125048, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 58212} +{"epoch": 0.50390625, "grad_norm": 0.6588469743728638, "learning_rate": 9.042669919862615e-05, "loss": 0.015841718763113022, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.01597, "step": 129, "tokens/total": 4156488, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 58676} +{"epoch": 0.5078125, "grad_norm": 1.2328574657440186, "learning_rate": 9.024696378701557e-05, "loss": 0.02290634997189045, "memory/device_reserved (GiB)": 66.81, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.02317, "step": 130, "tokens/total": 4189344, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 59109} +{"epoch": 0.51171875, "grad_norm": 0.33149951696395874, "learning_rate": 9.006576160536948e-05, "loss": 0.01335914433002472, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.01345, "step": 131, "tokens/total": 4220768, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 59558} +{"epoch": 0.515625, "grad_norm": 0.5537592172622681, "learning_rate": 8.988310019425035e-05, "loss": 0.015209494158625603, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.01533, "step": 132, "tokens/total": 4256880, "tokens/train_per_sec_per_gpu": 7.24, "tokens/trainable": 60002} +{"epoch": 0.51953125, "grad_norm": 0.40782561898231506, "learning_rate": 8.969898715494506e-05, "loss": 0.013169552199542522, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01326, "step": 133, "tokens/total": 4290928, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 60483} +{"epoch": 0.5234375, "grad_norm": 0.9064269065856934, "learning_rate": 8.951343014914869e-05, "loss": 0.021588170900940895, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.02182, "step": 134, "tokens/total": 4325728, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 60949} +{"epoch": 0.52734375, "grad_norm": 0.4814602732658386, "learning_rate": 8.932643689864568e-05, "loss": 0.002967614447697997, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00297, "step": 135, "tokens/total": 4358848, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 61436} +{"epoch": 0.53125, "grad_norm": 0.5476586222648621, "learning_rate": 8.913801518498845e-05, "loss": 0.0055268798023462296, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00554, "step": 136, "tokens/total": 4393504, "tokens/train_per_sec_per_gpu": 6.96, "tokens/trainable": 61897} +{"epoch": 0.53515625, "grad_norm": 0.5221210718154907, "learning_rate": 8.894817284917364e-05, "loss": 0.004225966986268759, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00423, "step": 137, "tokens/total": 4427912, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 62351} +{"epoch": 0.5390625, "grad_norm": 1.6795907020568848, "learning_rate": 8.875691779131569e-05, "loss": 0.06295324862003326, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.06498, "step": 138, "tokens/total": 4462448, "tokens/train_per_sec_per_gpu": 6.46, "tokens/trainable": 62783} +{"epoch": 0.54296875, "grad_norm": 0.39987248182296753, "learning_rate": 8.856425797031829e-05, "loss": 0.009205452166497707, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00925, "step": 139, "tokens/total": 4495992, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 63225} +{"epoch": 0.546875, "grad_norm": 0.4727531373500824, "learning_rate": 8.837020140354295e-05, "loss": 0.008560815826058388, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.0086, "step": 140, "tokens/total": 4529600, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 63690} +{"epoch": 0.55078125, "grad_norm": 0.9709975123405457, "learning_rate": 8.817475616647554e-05, "loss": 0.016477879136800766, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.01661, "step": 141, "tokens/total": 4562536, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 64157} +{"epoch": 0.5546875, "grad_norm": 0.8971356153488159, "learning_rate": 8.797793039239017e-05, "loss": 0.017687112092971802, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.01784, "step": 142, "tokens/total": 4595232, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 64605} +{"epoch": 0.55859375, "grad_norm": 1.725728988647461, "learning_rate": 8.777973227201069e-05, "loss": 0.01220017485320568, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.01227, "step": 143, "tokens/total": 4629712, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 65114} +{"epoch": 0.5625, "grad_norm": 0.5233529806137085, "learning_rate": 8.758017005316988e-05, "loss": 0.009625600650906563, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00967, "step": 144, "tokens/total": 4661752, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 65539} +{"epoch": 0.56640625, "grad_norm": 0.980044424533844, "learning_rate": 8.737925204046629e-05, "loss": 0.03837589547038078, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.03912, "step": 145, "tokens/total": 4694416, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 65989} +{"epoch": 0.5703125, "grad_norm": 0.5976966023445129, "learning_rate": 8.717698659491851e-05, "loss": 0.012167319655418396, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.01224, "step": 146, "tokens/total": 4728264, "tokens/train_per_sec_per_gpu": 7.09, "tokens/trainable": 66446} +{"epoch": 0.57421875, "grad_norm": 1.2948615550994873, "learning_rate": 8.697338213361735e-05, "loss": 0.01897997222840786, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.01916, "step": 147, "tokens/total": 4758952, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 66894} +{"epoch": 0.578125, "grad_norm": 0.6401638388633728, "learning_rate": 8.676844712937552e-05, "loss": 0.005718442145735025, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00573, "step": 148, "tokens/total": 4795624, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 67366} +{"epoch": 0.58203125, "grad_norm": 0.7857474088668823, "learning_rate": 8.656219011037509e-05, "loss": 0.013177258893847466, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01326, "step": 149, "tokens/total": 4828976, "tokens/train_per_sec_per_gpu": 6.65, "tokens/trainable": 67854} +{"epoch": 0.5859375, "grad_norm": 0.5574570298194885, "learning_rate": 8.63546196598125e-05, "loss": 0.007024134509265423, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00705, "step": 150, "tokens/total": 4862128, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 68302} +{"epoch": 0.58984375, "grad_norm": 0.3662789463996887, "learning_rate": 8.614574441554145e-05, "loss": 0.01086965948343277, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.01093, "step": 151, "tokens/total": 4892760, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 68727} +{"epoch": 0.59375, "grad_norm": 0.3210954964160919, "learning_rate": 8.593557306971349e-05, "loss": 0.004066256806254387, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00407, "step": 152, "tokens/total": 4925296, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 69182} +{"epoch": 0.59765625, "grad_norm": 0.24930381774902344, "learning_rate": 8.572411436841618e-05, "loss": 0.0030877720564603806, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00309, "step": 153, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 69657} +{"epoch": 0.6015625, "grad_norm": 0.3242975175380707, "learning_rate": 8.551137711130922e-05, "loss": 0.007609222549945116, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00764, "step": 154, "tokens/total": 4985520, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 70063} +{"epoch": 0.60546875, "grad_norm": 0.519197940826416, "learning_rate": 8.529737015125824e-05, "loss": 0.00504706148058176, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00506, "step": 155, "tokens/total": 5019320, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 70529} +{"epoch": 0.609375, "grad_norm": 1.4647190570831299, "learning_rate": 8.508210239396639e-05, "loss": 0.02667926624417305, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.02704, "step": 156, "tokens/total": 5052056, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 70978} +{"epoch": 0.61328125, "grad_norm": 0.12198933213949203, "learning_rate": 8.486558279760375e-05, "loss": 0.0010343643371015787, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00103, "step": 157, "tokens/total": 5082280, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 71404} +{"epoch": 0.6171875, "grad_norm": 0.584303617477417, "learning_rate": 8.464782037243449e-05, "loss": 0.014073871076107025, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01417, "step": 158, "tokens/total": 5113912, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 71845} +{"epoch": 0.62109375, "grad_norm": 0.8534519672393799, "learning_rate": 8.442882418044202e-05, "loss": 0.012789803557097912, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.01287, "step": 159, "tokens/total": 5145200, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 72322} +{"epoch": 0.625, "grad_norm": 3.3816781044006348, "learning_rate": 8.420860333495179e-05, "loss": 0.03351954370737076, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.03409, "step": 160, "tokens/total": 5174928, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 72737} +{"epoch": 0.62890625, "grad_norm": 0.08307597786188126, "learning_rate": 8.398716700025208e-05, "loss": 0.00021704615210182965, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00022, "step": 161, "tokens/total": 5206888, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 73189} +{"epoch": 0.6328125, "grad_norm": 1.6514892578125, "learning_rate": 8.376452439121266e-05, "loss": 0.02646343782544136, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.02682, "step": 162, "tokens/total": 5236336, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 73640} +{"epoch": 0.63671875, "grad_norm": 0.5291248559951782, "learning_rate": 8.354068477290124e-05, "loss": 0.006369778886437416, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00639, "step": 163, "tokens/total": 5267760, "tokens/train_per_sec_per_gpu": 10.22, "tokens/trainable": 74099} +{"epoch": 0.640625, "grad_norm": 1.628806710243225, "learning_rate": 8.331565746019807e-05, "loss": 0.010070119984447956, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.01012, "step": 164, "tokens/total": 5298896, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 74557} +{"epoch": 0.64453125, "grad_norm": 0.8932338356971741, "learning_rate": 8.308945181740812e-05, "loss": 0.024456266313791275, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.02476, "step": 165, "tokens/total": 5330792, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 74999} +{"epoch": 0.6484375, "grad_norm": 0.5134707093238831, "learning_rate": 8.286207725787153e-05, "loss": 0.008323743008077145, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00836, "step": 166, "tokens/total": 5364144, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 75444} +{"epoch": 0.65234375, "grad_norm": 1.286734938621521, "learning_rate": 8.263354324357182e-05, "loss": 0.03261196240782738, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.03315, "step": 167, "tokens/total": 5396656, "tokens/train_per_sec_per_gpu": 6.89, "tokens/trainable": 75879} +{"epoch": 0.65625, "grad_norm": 0.3120765686035156, "learning_rate": 8.240385928474219e-05, "loss": 0.01235003862529993, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.01243, "step": 168, "tokens/total": 5427736, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 76293} +{"epoch": 0.66015625, "grad_norm": 0.5225456953048706, "learning_rate": 8.217303493946967e-05, "loss": 0.005559472367167473, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00557, "step": 169, "tokens/total": 5461752, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 76741} +{"epoch": 0.6640625, "grad_norm": 0.6284992098808289, "learning_rate": 8.194107981329746e-05, "loss": 0.015661176294088364, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01578, "step": 170, "tokens/total": 5498688, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 77195} +{"epoch": 0.66796875, "grad_norm": 0.5901732444763184, "learning_rate": 8.170800355882518e-05, "loss": 0.0035855481401085854, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00359, "step": 171, "tokens/total": 5530936, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 77662} +{"epoch": 0.671875, "grad_norm": 0.7744209170341492, "learning_rate": 8.147381587530713e-05, "loss": 0.018622886389493942, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.0188, "step": 172, "tokens/total": 5564176, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 78116} +{"epoch": 0.67578125, "grad_norm": 0.8711299896240234, "learning_rate": 8.123852650824877e-05, "loss": 0.017497241497039795, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.01765, "step": 173, "tokens/total": 5596856, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 78599} +{"epoch": 0.6796875, "grad_norm": 0.37193232774734497, "learning_rate": 8.100214524900103e-05, "loss": 0.0066290562972426414, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00665, "step": 174, "tokens/total": 5630016, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 79030} +{"epoch": 0.68359375, "grad_norm": 0.10896377265453339, "learning_rate": 8.076468193435301e-05, "loss": 0.0015176089946180582, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00152, "step": 175, "tokens/total": 5659656, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 79454} +{"epoch": 0.6875, "grad_norm": 0.4318675696849823, "learning_rate": 8.052614644612253e-05, "loss": 0.00965411402285099, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.0097, "step": 176, "tokens/total": 5692408, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 79915} +{"epoch": 0.69140625, "grad_norm": 0.20827221870422363, "learning_rate": 8.028654871074489e-05, "loss": 0.0013974625617265701, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0014, "step": 177, "tokens/total": 5724672, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 80356} +{"epoch": 0.6953125, "grad_norm": 0.828665554523468, "learning_rate": 8.004589869885986e-05, "loss": 0.008697351440787315, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00874, "step": 178, "tokens/total": 5754384, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 80774} +{"epoch": 0.69921875, "grad_norm": 0.5381302833557129, "learning_rate": 7.980420642489674e-05, "loss": 0.013426994904875755, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01352, "step": 179, "tokens/total": 5789344, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 81247} +{"epoch": 0.703125, "grad_norm": 0.1781156361103058, "learning_rate": 7.95614819466576e-05, "loss": 0.0025504138320684433, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00255, "step": 180, "tokens/total": 5821816, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 81713} +{"epoch": 0.70703125, "grad_norm": 0.8328827023506165, "learning_rate": 7.931773536489872e-05, "loss": 0.0038023339584469795, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00381, "step": 181, "tokens/total": 5854168, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 82158} +{"epoch": 0.7109375, "grad_norm": 0.5913953185081482, "learning_rate": 7.907297682291035e-05, "loss": 0.005937093403190374, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00595, "step": 182, "tokens/total": 5886064, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 82621} +{"epoch": 0.71484375, "grad_norm": 0.5973328948020935, "learning_rate": 7.882721650609442e-05, "loss": 0.01583988405764103, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.01597, "step": 183, "tokens/total": 5921648, "tokens/train_per_sec_per_gpu": 6.77, "tokens/trainable": 83122} +{"epoch": 0.71875, "grad_norm": 0.443663626909256, "learning_rate": 7.85804646415409e-05, "loss": 0.0050047351978719234, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.00502, "step": 184, "tokens/total": 5954952, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 83578} +{"epoch": 0.72265625, "grad_norm": 2.0329601764678955, "learning_rate": 7.833273149760207e-05, "loss": 0.0023097917437553406, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00231, "step": 185, "tokens/total": 5985432, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 84048} +{"epoch": 0.7265625, "grad_norm": 1.0991514921188354, "learning_rate": 7.808402738346527e-05, "loss": 0.02019963040947914, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.02041, "step": 186, "tokens/total": 6020664, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 84497} +{"epoch": 0.73046875, "grad_norm": 0.1699441522359848, "learning_rate": 7.783436264872382e-05, "loss": 0.0010285498574376106, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00103, "step": 187, "tokens/total": 6055024, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 84983} +{"epoch": 0.734375, "grad_norm": 0.231630340218544, "learning_rate": 7.758374768294647e-05, "loss": 0.00189986324403435, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.0019, "step": 188, "tokens/total": 6088296, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 85462} +{"epoch": 0.73828125, "grad_norm": 0.07221562415361404, "learning_rate": 7.733219291524489e-05, "loss": 0.0011329654371365905, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00113, "step": 189, "tokens/total": 6119840, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 85892} +{"epoch": 0.7421875, "grad_norm": 0.1729746162891388, "learning_rate": 7.707970881383977e-05, "loss": 0.001901506562717259, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.0019, "step": 190, "tokens/total": 6149880, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 86341} +{"epoch": 0.74609375, "grad_norm": 0.38091906905174255, "learning_rate": 7.682630588562518e-05, "loss": 0.006689479574561119, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00671, "step": 191, "tokens/total": 6182904, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 86772} +{"epoch": 0.75, "grad_norm": 0.41781824827194214, "learning_rate": 7.657199467573129e-05, "loss": 0.008741401135921478, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00878, "step": 192, "tokens/total": 6214144, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 87244} +{"epoch": 0.75390625, "grad_norm": 0.46491310000419617, "learning_rate": 7.631678576708561e-05, "loss": 0.015157745219767094, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.01527, "step": 193, "tokens/total": 6247072, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 87736} +{"epoch": 0.7578125, "grad_norm": 0.799431324005127, "learning_rate": 7.606068977997255e-05, "loss": 0.007725500501692295, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00776, "step": 194, "tokens/total": 6281112, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 88214} +{"epoch": 0.76171875, "grad_norm": 0.9036203026771545, "learning_rate": 7.580371737159148e-05, "loss": 0.010495302267372608, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.01055, "step": 195, "tokens/total": 6310200, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 88625} +{"epoch": 0.765625, "grad_norm": 1.028168797492981, "learning_rate": 7.554587923561324e-05, "loss": 0.010172421112656593, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.01022, "step": 196, "tokens/total": 6341792, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 89063} +{"epoch": 0.76953125, "grad_norm": 0.30308812856674194, "learning_rate": 7.528718610173511e-05, "loss": 0.003554708557203412, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.00356, "step": 197, "tokens/total": 6374040, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 89484} +{"epoch": 0.7734375, "grad_norm": 0.07448142021894455, "learning_rate": 7.502764873523431e-05, "loss": 0.0009865346364676952, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00099, "step": 198, "tokens/total": 6405552, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 89922} +{"epoch": 0.77734375, "grad_norm": 0.37261074781417847, "learning_rate": 7.476727793652011e-05, "loss": 0.0078851617872715, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00792, "step": 199, "tokens/total": 6436016, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 90401} +{"epoch": 0.78125, "grad_norm": 0.9740132689476013, "learning_rate": 7.450608454068415e-05, "loss": 0.02247234247624874, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.02273, "step": 200, "tokens/total": 6468656, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 90842} +{"epoch": 0.78515625, "grad_norm": 0.27807632088661194, "learning_rate": 7.424407941704987e-05, "loss": 0.010603494010865688, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.01066, "step": 201, "tokens/total": 6502728, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 91343} +{"epoch": 0.7890625, "grad_norm": 0.16687028110027313, "learning_rate": 7.398127346871986e-05, "loss": 0.004106136970221996, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00411, "step": 202, "tokens/total": 6535616, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 91798} +{"epoch": 0.79296875, "grad_norm": 0.09061331301927567, "learning_rate": 7.371767763212238e-05, "loss": 0.0019848537631332874, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00199, "step": 203, "tokens/total": 6567088, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 92251} +{"epoch": 0.796875, "grad_norm": 0.103018619120121, "learning_rate": 7.345330287655617e-05, "loss": 0.0025630262680351734, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00257, "step": 204, "tokens/total": 6600808, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 92714} +{"epoch": 0.80078125, "grad_norm": 0.30150535702705383, "learning_rate": 7.31881602037339e-05, "loss": 0.008359881117939949, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.00839, "step": 205, "tokens/total": 6631384, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 93154} +{"epoch": 0.8046875, "grad_norm": 0.17489802837371826, "learning_rate": 7.29222606473245e-05, "loss": 0.001929061720147729, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00193, "step": 206, "tokens/total": 6661888, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 93599} +{"epoch": 0.80859375, "grad_norm": 0.0836346447467804, "learning_rate": 7.265561527249383e-05, "loss": 0.0013007845263928175, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.0013, "step": 207, "tokens/total": 6693456, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 94036} +{"epoch": 0.8125, "grad_norm": 0.36081165075302124, "learning_rate": 7.238823517544436e-05, "loss": 0.011022322811186314, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01108, "step": 208, "tokens/total": 6727840, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 94506} +{"epoch": 0.81640625, "grad_norm": 0.10691730678081512, "learning_rate": 7.212013148295333e-05, "loss": 0.0012345437426120043, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00124, "step": 209, "tokens/total": 6761704, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 94988} +{"epoch": 0.8203125, "grad_norm": 0.7793547511100769, "learning_rate": 7.185131535190975e-05, "loss": 0.03548331558704376, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.03612, "step": 210, "tokens/total": 6792648, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 95497} +{"epoch": 0.82421875, "grad_norm": 0.4421168863773346, "learning_rate": 7.158179796885005e-05, "loss": 0.009851559065282345, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.0099, "step": 211, "tokens/total": 6824424, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 95948} +{"epoch": 0.828125, "grad_norm": 0.02446850575506687, "learning_rate": 7.131159054949273e-05, "loss": 0.0004902243963442743, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00049, "step": 212, "tokens/total": 6861216, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 96422} +{"epoch": 0.83203125, "grad_norm": 0.134645476937294, "learning_rate": 7.104070433827139e-05, "loss": 0.0018444882007315755, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00185, "step": 213, "tokens/total": 6892704, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 96821} +{"epoch": 0.8359375, "grad_norm": 0.3040945827960968, "learning_rate": 7.076915060786705e-05, "loss": 0.0012101758038625121, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.00121, "step": 214, "tokens/total": 6924816, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 97263} +{"epoch": 0.83984375, "grad_norm": 0.28499874472618103, "learning_rate": 7.049694065873882e-05, "loss": 0.0056753866374492645, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00569, "step": 215, "tokens/total": 6957536, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 97720} +{"epoch": 0.84375, "grad_norm": 1.2875065803527832, "learning_rate": 7.022408581865382e-05, "loss": 0.014887592755258083, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.015, "step": 216, "tokens/total": 6991448, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 98194} +{"epoch": 0.84765625, "grad_norm": 0.08454510569572449, "learning_rate": 6.99505974422157e-05, "loss": 0.0009554304997436702, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00096, "step": 217, "tokens/total": 7023712, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 98667} +{"epoch": 0.8515625, "grad_norm": 0.257866770029068, "learning_rate": 6.967648691039213e-05, "loss": 0.0059758927673101425, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00599, "step": 218, "tokens/total": 7055040, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 99100} +{"epoch": 0.85546875, "grad_norm": 0.10068531334400177, "learning_rate": 6.940176563004123e-05, "loss": 0.0018341203685849905, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00184, "step": 219, "tokens/total": 7084944, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 99532} +{"epoch": 0.859375, "grad_norm": 0.19049163162708282, "learning_rate": 6.912644503343682e-05, "loss": 0.0043629673309624195, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00437, "step": 220, "tokens/total": 7119264, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 99970} +{"epoch": 0.86328125, "grad_norm": 0.13036930561065674, "learning_rate": 6.885053657779273e-05, "loss": 0.0023682827595621347, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.00237, "step": 221, "tokens/total": 7150848, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 100442} +{"epoch": 0.8671875, "grad_norm": 0.08179955184459686, "learning_rate": 6.857405174478604e-05, "loss": 0.001103952876292169, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.0011, "step": 222, "tokens/total": 7185176, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 100895} +{"epoch": 0.87109375, "grad_norm": 0.20766468346118927, "learning_rate": 6.82970020400792e-05, "loss": 0.0023505627177655697, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00235, "step": 223, "tokens/total": 7219024, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 101342} +{"epoch": 0.875, "grad_norm": 0.30998510122299194, "learning_rate": 6.801939899284132e-05, "loss": 0.002843407681211829, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00285, "step": 224, "tokens/total": 7253816, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 101848} +{"epoch": 0.87890625, "grad_norm": 0.23728157579898834, "learning_rate": 6.774125415526827e-05, "loss": 0.001585409278050065, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00159, "step": 225, "tokens/total": 7286344, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 102319} +{"epoch": 0.8828125, "grad_norm": 0.16891424357891083, "learning_rate": 6.746257910210214e-05, "loss": 0.001733267679810524, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00173, "step": 226, "tokens/total": 7317952, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 102769} +{"epoch": 0.88671875, "grad_norm": 0.5616126656532288, "learning_rate": 6.718338543014937e-05, "loss": 0.015175464563071728, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.01529, "step": 227, "tokens/total": 7347336, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 103219} +{"epoch": 0.890625, "grad_norm": 1.6422094106674194, "learning_rate": 6.69036847577983e-05, "loss": 0.0077469125390052795, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00778, "step": 228, "tokens/total": 7377592, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 103675} +{"epoch": 0.89453125, "grad_norm": 0.06853199750185013, "learning_rate": 6.662348872453553e-05, "loss": 0.0005721059860661626, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00057, "step": 229, "tokens/total": 7410184, "tokens/train_per_sec_per_gpu": 10.56, "tokens/trainable": 104136} +{"epoch": 0.8984375, "grad_norm": 0.00430182833224535, "learning_rate": 6.63428089904618e-05, "loss": 2.6206422262475826e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00003, "step": 230, "tokens/total": 7443984, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 104588} +{"epoch": 0.90234375, "grad_norm": 1.3689031600952148, "learning_rate": 6.60616572358065e-05, "loss": 0.027974585071206093, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.02837, "step": 231, "tokens/total": 7472480, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 104988} +{"epoch": 0.90625, "grad_norm": 0.10290393233299255, "learning_rate": 6.578004516044172e-05, "loss": 0.000632286595646292, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00063, "step": 232, "tokens/total": 7505992, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 105455} +{"epoch": 0.91015625, "grad_norm": 0.010829917155206203, "learning_rate": 6.549798448339548e-05, "loss": 0.00010976717749144882, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00011, "step": 233, "tokens/total": 7541168, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 105918} +{"epoch": 0.9140625, "grad_norm": 0.07901016622781754, "learning_rate": 6.521548694236384e-05, "loss": 0.0007423296920023859, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00074, "step": 234, "tokens/total": 7574208, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 106355} +{"epoch": 0.91796875, "grad_norm": 0.1453685462474823, "learning_rate": 6.493256429322259e-05, "loss": 0.0017543069552630186, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00176, "step": 235, "tokens/total": 7607016, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 106786} +{"epoch": 0.921875, "grad_norm": 0.04380859434604645, "learning_rate": 6.464922830953799e-05, "loss": 0.0005746890674345195, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00057, "step": 236, "tokens/total": 7641744, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 107219} +{"epoch": 0.92578125, "grad_norm": 0.48180097341537476, "learning_rate": 6.436549078207688e-05, "loss": 0.020476190373301506, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.02069, "step": 237, "tokens/total": 7673096, "tokens/train_per_sec_per_gpu": 10.16, "tokens/trainable": 107679} +{"epoch": 0.9296875, "grad_norm": 0.2439500093460083, "learning_rate": 6.408136351831592e-05, "loss": 0.003113374812528491, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00312, "step": 238, "tokens/total": 7705344, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 108119} +{"epoch": 0.93359375, "grad_norm": 0.04133140295743942, "learning_rate": 6.379685834195036e-05, "loss": 0.0003982940979767591, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.0004, "step": 239, "tokens/total": 7740240, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 108626} +{"epoch": 0.9375, "grad_norm": 0.061062805354595184, "learning_rate": 6.351198709240186e-05, "loss": 0.0007701229769736528, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00077, "step": 240, "tokens/total": 7775392, "tokens/train_per_sec_per_gpu": 7.12, "tokens/trainable": 109079} +{"epoch": 0.94140625, "grad_norm": 0.3559311032295227, "learning_rate": 6.32267616243259e-05, "loss": 0.004487609025090933, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.0045, "step": 241, "tokens/total": 7807296, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 109542} +{"epoch": 0.9453125, "grad_norm": 0.05696408078074455, "learning_rate": 6.294119380711849e-05, "loss": 0.001115255057811737, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00112, "step": 242, "tokens/total": 7838200, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 109969} +{"epoch": 0.94921875, "grad_norm": 1.249733805656433, "learning_rate": 6.265529552442209e-05, "loss": 0.004159228876233101, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00417, "step": 243, "tokens/total": 7870936, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 110431} +{"epoch": 0.953125, "grad_norm": 0.028612326830625534, "learning_rate": 6.236907867363127e-05, "loss": 0.00047021356294862926, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00047, "step": 244, "tokens/total": 7902512, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 110877} +{"epoch": 0.95703125, "grad_norm": 0.2221945822238922, "learning_rate": 6.208255516539749e-05, "loss": 0.0018850911874324083, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00189, "step": 245, "tokens/total": 7933224, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 111350} +{"epoch": 0.9609375, "grad_norm": 0.16259577870368958, "learning_rate": 6.179573692313344e-05, "loss": 0.0023871229495853186, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00239, "step": 246, "tokens/total": 7968456, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 111798} +{"epoch": 0.96484375, "grad_norm": 0.14943869411945343, "learning_rate": 6.150863588251694e-05, "loss": 0.004580756649374962, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00459, "step": 247, "tokens/total": 8003592, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 112264} +{"epoch": 0.96875, "grad_norm": 0.8091598749160767, "learning_rate": 6.122126399099419e-05, "loss": 0.00870725978165865, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00875, "step": 248, "tokens/total": 8040048, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 112772} +{"epoch": 0.97265625, "grad_norm": 0.15346720814704895, "learning_rate": 6.0933633207282615e-05, "loss": 0.0021038998384028673, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00211, "step": 249, "tokens/total": 8074000, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 113278} +{"epoch": 0.9765625, "grad_norm": 0.535961389541626, "learning_rate": 6.064575550087316e-05, "loss": 0.003811957547441125, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00382, "step": 250, "tokens/total": 8108304, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 113716} +{"epoch": 0.98046875, "grad_norm": 14.225587844848633, "learning_rate": 6.0357642851532245e-05, "loss": 0.005195010453462601, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00521, "step": 251, "tokens/total": 8143464, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 114207} +{"epoch": 0.984375, "grad_norm": 0.11795674264431, "learning_rate": 6.0069307248803294e-05, "loss": 0.0017574019730091095, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00176, "step": 252, "tokens/total": 8175576, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 114628} +{"epoch": 0.98828125, "grad_norm": 0.9705619812011719, "learning_rate": 5.9780760691507635e-05, "loss": 0.01564965955913067, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.01577, "step": 253, "tokens/total": 8205528, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 115060} +{"epoch": 0.9921875, "grad_norm": 0.2945167124271393, "learning_rate": 5.9492015187245334e-05, "loss": 0.002195490524172783, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.0022, "step": 254, "tokens/total": 8239424, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 115514} +{"epoch": 0.99609375, "grad_norm": 0.06519181281328201, "learning_rate": 5.920308275189541e-05, "loss": 0.000992052722722292, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00099, "step": 255, "tokens/total": 8270392, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 115915} +{"epoch": 1.0, "grad_norm": 0.13420285284519196, "learning_rate": 5.8913975409115874e-05, "loss": 0.0013911501737311482, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00139, "step": 256, "tokens/total": 8302096, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 116378} +{"epoch": 1.00390625, "grad_norm": 0.024327946826815605, "learning_rate": 5.8624705189843395e-05, "loss": 0.00023767948732711375, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00024, "step": 257, "tokens/total": 8337064, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 116865} +{"epoch": 1.0078125, "grad_norm": 0.03079986944794655, "learning_rate": 5.833528413179249e-05, "loss": 0.0004074184689670801, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00041, "step": 258, "tokens/total": 8372840, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 117363} +{"epoch": 1.01171875, "grad_norm": 0.27850818634033203, "learning_rate": 5.80457242789548e-05, "loss": 0.0021863081492483616, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00219, "step": 259, "tokens/total": 8409728, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 117860} +{"epoch": 1.015625, "grad_norm": 0.39357075095176697, "learning_rate": 5.77560376810977e-05, "loss": 0.006359363906085491, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.00638, "step": 260, "tokens/total": 8442296, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 118324} +{"epoch": 1.01953125, "grad_norm": 0.06336144357919693, "learning_rate": 5.7466236393263005e-05, "loss": 0.00012174437870271504, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00012, "step": 261, "tokens/total": 8474568, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 118770} +{"epoch": 1.0234375, "grad_norm": 0.22898761928081512, "learning_rate": 5.717633247526522e-05, "loss": 0.0017845594557002187, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00179, "step": 262, "tokens/total": 8505520, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 119206} +{"epoch": 1.02734375, "grad_norm": 0.01268986240029335, "learning_rate": 5.688633799118971e-05, "loss": 0.0001614296343177557, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00016, "step": 263, "tokens/total": 8535064, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 119610} +{"epoch": 1.03125, "grad_norm": 0.28533855080604553, "learning_rate": 5.659626500889066e-05, "loss": 0.0026781519409269094, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00268, "step": 264, "tokens/total": 8563600, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 120080} +{"epoch": 1.03515625, "grad_norm": 0.017311325296759605, "learning_rate": 5.6306125599488905e-05, "loss": 0.00019131893350277096, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00019, "step": 265, "tokens/total": 8592648, "tokens/train_per_sec_per_gpu": 9.88, "tokens/trainable": 120519} +{"epoch": 1.0390625, "grad_norm": 0.10480345785617828, "learning_rate": 5.601593183686955e-05, "loss": 0.0006607671966776252, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00066, "step": 266, "tokens/total": 8627328, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 120995} +{"epoch": 1.04296875, "grad_norm": 0.06213809549808502, "learning_rate": 5.572569579717961e-05, "loss": 0.00047296658158302307, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00047, "step": 267, "tokens/total": 8659760, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 121459} +{"epoch": 1.046875, "grad_norm": 0.0022871270775794983, "learning_rate": 5.543542955832538e-05, "loss": 4.1425802919548005e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00004, "step": 268, "tokens/total": 8696968, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 121902} +{"epoch": 1.05078125, "grad_norm": 0.8113918304443359, "learning_rate": 5.514514519946986e-05, "loss": 0.008341900072991848, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00838, "step": 269, "tokens/total": 8735392, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 122399} +{"epoch": 1.0546875, "grad_norm": 0.07688327878713608, "learning_rate": 5.485485480053015e-05, "loss": 0.00073475536191836, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00074, "step": 270, "tokens/total": 8769840, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 122804} +{"epoch": 1.05859375, "grad_norm": 0.08790811896324158, "learning_rate": 5.4564570441674645e-05, "loss": 0.0004823724739253521, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00048, "step": 271, "tokens/total": 8800648, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 123244} +{"epoch": 1.0625, "grad_norm": 0.013212495483458042, "learning_rate": 5.42743042028204e-05, "loss": 0.0001724237808957696, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00017, "step": 272, "tokens/total": 8832480, "tokens/train_per_sec_per_gpu": 6.81, "tokens/trainable": 123683} +{"epoch": 1.06640625, "grad_norm": 0.06787249445915222, "learning_rate": 5.3984068163130464e-05, "loss": 0.0005260569741949439, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00053, "step": 273, "tokens/total": 8864840, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 124153} +{"epoch": 1.0703125, "grad_norm": 0.3362126350402832, "learning_rate": 5.369387440051111e-05, "loss": 0.006673148833215237, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.0067, "step": 274, "tokens/total": 8897272, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 124616} +{"epoch": 1.07421875, "grad_norm": 0.004026994574815035, "learning_rate": 5.340373499110935e-05, "loss": 4.768322469317354e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00005, "step": 275, "tokens/total": 8930688, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 125109} +{"epoch": 1.078125, "grad_norm": 0.0057481518015265465, "learning_rate": 5.3113662008810304e-05, "loss": 4.733963214675896e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00005, "step": 276, "tokens/total": 8967216, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 125591} +{"epoch": 1.08203125, "grad_norm": 0.3589629530906677, "learning_rate": 5.282366752473479e-05, "loss": 0.002558526350185275, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00256, "step": 277, "tokens/total": 9001368, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 126065} +{"epoch": 1.0859375, "grad_norm": 0.02042149007320404, "learning_rate": 5.2533763606737005e-05, "loss": 0.00016035082808230072, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00016, "step": 278, "tokens/total": 9036808, "tokens/train_per_sec_per_gpu": 6.91, "tokens/trainable": 126480} +{"epoch": 1.08984375, "grad_norm": 0.005575600080192089, "learning_rate": 5.224396231890232e-05, "loss": 5.454570418805815e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00005, "step": 279, "tokens/total": 9067840, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 126929} +{"epoch": 1.09375, "grad_norm": 0.042009662836790085, "learning_rate": 5.195427572104522e-05, "loss": 0.00046339206164702773, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00046, "step": 280, "tokens/total": 9098936, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 127366} +{"epoch": 1.09765625, "grad_norm": 0.27615460753440857, "learning_rate": 5.166471586820751e-05, "loss": 0.0033168341033160686, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00332, "step": 281, "tokens/total": 9133264, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 127831} +{"epoch": 1.1015625, "grad_norm": 0.27492815256118774, "learning_rate": 5.1375294810156615e-05, "loss": 0.0016835009446367621, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00168, "step": 282, "tokens/total": 9165008, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 128278} +{"epoch": 1.10546875, "grad_norm": 0.011548320762813091, "learning_rate": 5.1086024590884144e-05, "loss": 0.00015508648357354105, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00016, "step": 283, "tokens/total": 9197952, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 128727} +{"epoch": 1.109375, "grad_norm": 0.009876815602183342, "learning_rate": 5.079691724810461e-05, "loss": 5.134270031703636e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00005, "step": 284, "tokens/total": 9232576, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 129162} +{"epoch": 1.11328125, "grad_norm": 0.07792297005653381, "learning_rate": 5.0507984812754684e-05, "loss": 0.000799264176748693, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.0008, "step": 285, "tokens/total": 9264568, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 129659} +{"epoch": 1.1171875, "grad_norm": 0.16623856127262115, "learning_rate": 5.021923930849237e-05, "loss": 0.0015882370062172413, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00159, "step": 286, "tokens/total": 9296544, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 130145} +{"epoch": 1.12109375, "grad_norm": 0.005461766384541988, "learning_rate": 4.99306927511967e-05, "loss": 2.912526724685449e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00003, "step": 287, "tokens/total": 9327936, "tokens/train_per_sec_per_gpu": 10.47, "tokens/trainable": 130585} +{"epoch": 1.125, "grad_norm": 0.5486096739768982, "learning_rate": 4.964235714846775e-05, "loss": 0.0017560584237799048, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00176, "step": 288, "tokens/total": 9357984, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 131005} +{"epoch": 1.12890625, "grad_norm": 0.002947766799479723, "learning_rate": 4.9354244499126866e-05, "loss": 2.0111407138756476e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00002, "step": 289, "tokens/total": 9388664, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 131471} +{"epoch": 1.1328125, "grad_norm": 0.006945752073079348, "learning_rate": 4.90663667927174e-05, "loss": 6.251640297705308e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00006, "step": 290, "tokens/total": 9420504, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 131920} +{"epoch": 1.13671875, "grad_norm": 0.07541875541210175, "learning_rate": 4.877873600900581e-05, "loss": 0.0002367179695283994, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00024, "step": 291, "tokens/total": 9452664, "tokens/train_per_sec_per_gpu": 6.32, "tokens/trainable": 132379} +{"epoch": 1.140625, "grad_norm": 0.012901903130114079, "learning_rate": 4.849136411748306e-05, "loss": 3.641889998107217e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00004, "step": 292, "tokens/total": 9488896, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 132851} +{"epoch": 1.14453125, "grad_norm": 0.00498847896233201, "learning_rate": 4.8204263076866574e-05, "loss": 3.098423621850088e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00003, "step": 293, "tokens/total": 9522112, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 133351} +{"epoch": 1.1484375, "grad_norm": 0.002539500594139099, "learning_rate": 4.791744483460251e-05, "loss": 3.467123315203935e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00003, "step": 294, "tokens/total": 9555632, "tokens/train_per_sec_per_gpu": 6.6, "tokens/trainable": 133784} +{"epoch": 1.15234375, "grad_norm": 0.0010344006586819887, "learning_rate": 4.7630921326368736e-05, "loss": 1.3255835256131832e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00001, "step": 295, "tokens/total": 9590904, "tokens/train_per_sec_per_gpu": 6.44, "tokens/trainable": 134209} +{"epoch": 1.15625, "grad_norm": 0.0022004619240760803, "learning_rate": 4.7344704475577916e-05, "loss": 1.1194908438483253e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00001, "step": 296, "tokens/total": 9620288, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 134645} +{"epoch": 1.16015625, "grad_norm": 0.13483761250972748, "learning_rate": 4.705880619288153e-05, "loss": 0.0013330458896234632, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00133, "step": 297, "tokens/total": 9651984, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 135047} +{"epoch": 1.1640625, "grad_norm": 0.009615610353648663, "learning_rate": 4.677323837567412e-05, "loss": 0.00010291516809957102, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.0001, "step": 298, "tokens/total": 9681368, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 135474} +{"epoch": 1.16796875, "grad_norm": 0.009708981961011887, "learning_rate": 4.6488012907598146e-05, "loss": 4.0767266909824684e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00004, "step": 299, "tokens/total": 9712872, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 135948} +{"epoch": 1.171875, "grad_norm": 0.5829634666442871, "learning_rate": 4.620314165804964e-05, "loss": 0.004702132195234299, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00471, "step": 300, "tokens/total": 9745184, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 136421} +{"epoch": 1.17578125, "grad_norm": 0.014927733689546585, "learning_rate": 4.591863648168407e-05, "loss": 7.35334397177212e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00007, "step": 301, "tokens/total": 9778928, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 136886} +{"epoch": 1.1796875, "grad_norm": 0.060300618410110474, "learning_rate": 4.5634509217923135e-05, "loss": 0.0001875467860372737, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00019, "step": 302, "tokens/total": 9810752, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 137321} +{"epoch": 1.18359375, "grad_norm": 0.837296187877655, "learning_rate": 4.535077169046201e-05, "loss": 0.006487594451755285, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00651, "step": 303, "tokens/total": 9843672, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 137758} +{"epoch": 1.1875, "grad_norm": 0.8695955276489258, "learning_rate": 4.506743570677743e-05, "loss": 0.013874595053493977, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.01397, "step": 304, "tokens/total": 9873752, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 138190} +{"epoch": 1.19140625, "grad_norm": 0.08198579400777817, "learning_rate": 4.478451305763618e-05, "loss": 0.0004458600305952132, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00045, "step": 305, "tokens/total": 9906272, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 138650} +{"epoch": 1.1953125, "grad_norm": 0.0017116469098255038, "learning_rate": 4.450201551660454e-05, "loss": 1.3419146853266284e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00001, "step": 306, "tokens/total": 9936296, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 139082} +{"epoch": 1.19921875, "grad_norm": 0.4639979898929596, "learning_rate": 4.4219954839558276e-05, "loss": 0.001729625160805881, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00173, "step": 307, "tokens/total": 9967720, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 139523} +{"epoch": 1.203125, "grad_norm": 0.10767651349306107, "learning_rate": 4.393834276419352e-05, "loss": 0.0004280200810171664, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.00043, "step": 308, "tokens/total": 9997176, "tokens/train_per_sec_per_gpu": 10.82, "tokens/trainable": 140010} +{"epoch": 1.20703125, "grad_norm": 0.03409081697463989, "learning_rate": 4.36571910095382e-05, "loss": 0.00023491538013331592, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00023, "step": 309, "tokens/total": 10031872, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 140454} +{"epoch": 1.2109375, "grad_norm": 0.6082446575164795, "learning_rate": 4.337651127546448e-05, "loss": 0.002238150220364332, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00224, "step": 310, "tokens/total": 10065240, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 140905} +{"epoch": 1.21484375, "grad_norm": 0.17172321677207947, "learning_rate": 4.3096315242201736e-05, "loss": 0.0011521242558956146, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00115, "step": 311, "tokens/total": 10096640, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 141339} +{"epoch": 1.21875, "grad_norm": 0.000603138527367264, "learning_rate": 4.2816614569850635e-05, "loss": 1.2262691598152742e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00001, "step": 312, "tokens/total": 10130192, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 141778} +{"epoch": 1.22265625, "grad_norm": 0.7030205726623535, "learning_rate": 4.2537420897897864e-05, "loss": 0.007254481315612793, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00728, "step": 313, "tokens/total": 10164248, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 142255} +{"epoch": 1.2265625, "grad_norm": 0.438889741897583, "learning_rate": 4.225874584473174e-05, "loss": 0.0037673290353268385, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00377, "step": 314, "tokens/total": 10194664, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 142693} +{"epoch": 1.23046875, "grad_norm": 0.15425051748752594, "learning_rate": 4.19806010071587e-05, "loss": 0.0004931662115268409, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00049, "step": 315, "tokens/total": 10227768, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 143157} +{"epoch": 1.234375, "grad_norm": 0.015639331191778183, "learning_rate": 4.170299795992081e-05, "loss": 9.91377528407611e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.0001, "step": 316, "tokens/total": 10260464, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 143603} +{"epoch": 1.23828125, "grad_norm": 0.023779258131980896, "learning_rate": 4.142594825521398e-05, "loss": 0.00021671778813470155, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00022, "step": 317, "tokens/total": 10297528, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 144084} +{"epoch": 1.2421875, "grad_norm": 0.2382124960422516, "learning_rate": 4.114946342220728e-05, "loss": 0.0010285875760018826, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00103, "step": 318, "tokens/total": 10331448, "tokens/train_per_sec_per_gpu": 7.0, "tokens/trainable": 144532} +{"epoch": 1.24609375, "grad_norm": 0.024649642407894135, "learning_rate": 4.087355496656321e-05, "loss": 0.00022051228734198958, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00022, "step": 319, "tokens/total": 10363152, "tokens/train_per_sec_per_gpu": 10.19, "tokens/trainable": 145004} +{"epoch": 1.25, "grad_norm": 0.4492398798465729, "learning_rate": 4.05982343699588e-05, "loss": 0.011294858530163765, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.01136, "step": 320, "tokens/total": 10397832, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 145498} +{"epoch": 1.25390625, "grad_norm": 0.012460244819521904, "learning_rate": 4.0323513089607876e-05, "loss": 0.00013070092245470732, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00013, "step": 321, "tokens/total": 10431416, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 145965} +{"epoch": 1.2578125, "grad_norm": 0.06486064940690994, "learning_rate": 4.004940255778431e-05, "loss": 0.0002918159880209714, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00029, "step": 322, "tokens/total": 10467384, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 146409} +{"epoch": 1.26171875, "grad_norm": 0.4662855565547943, "learning_rate": 3.977591418134619e-05, "loss": 0.0010276095708832145, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00103, "step": 323, "tokens/total": 10500792, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 146849} +{"epoch": 1.265625, "grad_norm": 0.4485973119735718, "learning_rate": 3.95030593412612e-05, "loss": 0.011318034492433071, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.01138, "step": 324, "tokens/total": 10532592, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 147272} +{"epoch": 1.26953125, "grad_norm": 0.0321507453918457, "learning_rate": 3.923084939213296e-05, "loss": 0.00040098922909237444, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.0004, "step": 325, "tokens/total": 10562392, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 147694} +{"epoch": 1.2734375, "grad_norm": 0.0951453447341919, "learning_rate": 3.895929566172861e-05, "loss": 0.0006329758325591683, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00063, "step": 326, "tokens/total": 10595416, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 148158} +{"epoch": 1.27734375, "grad_norm": 0.6256328225135803, "learning_rate": 3.868840945050728e-05, "loss": 0.005841926671564579, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00586, "step": 327, "tokens/total": 10630504, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 148590} +{"epoch": 1.28125, "grad_norm": 0.07340122014284134, "learning_rate": 3.841820203114995e-05, "loss": 0.00026048184372484684, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00026, "step": 328, "tokens/total": 10661632, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 149037} +{"epoch": 1.28515625, "grad_norm": 0.1546180695295334, "learning_rate": 3.814868464809027e-05, "loss": 0.001290036947466433, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00129, "step": 329, "tokens/total": 10691544, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 149439} +{"epoch": 1.2890625, "grad_norm": 0.049871284514665604, "learning_rate": 3.787986851704667e-05, "loss": 0.00031217848300002515, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00031, "step": 330, "tokens/total": 10724768, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 149869} +{"epoch": 1.29296875, "grad_norm": 0.046399276703596115, "learning_rate": 3.7611764824555654e-05, "loss": 0.0003146310045849532, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00031, "step": 331, "tokens/total": 10756008, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 150343} +{"epoch": 1.296875, "grad_norm": 0.3273402452468872, "learning_rate": 3.734438472750619e-05, "loss": 0.0024169739335775375, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00242, "step": 332, "tokens/total": 10793152, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 150814} +{"epoch": 1.30078125, "grad_norm": 0.07280176877975464, "learning_rate": 3.707773935267552e-05, "loss": 0.0004832536797039211, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00048, "step": 333, "tokens/total": 10827624, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 151300} +{"epoch": 1.3046875, "grad_norm": 0.036794234067201614, "learning_rate": 3.68118397962661e-05, "loss": 0.00041359811439178884, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00041, "step": 334, "tokens/total": 10859712, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 151708} +{"epoch": 1.30859375, "grad_norm": 0.011616821400821209, "learning_rate": 3.654669712344384e-05, "loss": 0.00012987773516215384, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00013, "step": 335, "tokens/total": 10888792, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 152164} +{"epoch": 1.3125, "grad_norm": 0.3637125492095947, "learning_rate": 3.628232236787763e-05, "loss": 0.003892817534506321, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.0039, "step": 336, "tokens/total": 10922968, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 152603} +{"epoch": 1.31640625, "grad_norm": 0.008042808622121811, "learning_rate": 3.6018726531280144e-05, "loss": 8.444848208455369e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00008, "step": 337, "tokens/total": 10958000, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 153094} +{"epoch": 1.3203125, "grad_norm": 1.343416452407837, "learning_rate": 3.575592058295017e-05, "loss": 0.007792562246322632, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00782, "step": 338, "tokens/total": 10989872, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 153573} +{"epoch": 1.32421875, "grad_norm": 0.017623450607061386, "learning_rate": 3.549391545931585e-05, "loss": 0.00010479827324161306, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.0001, "step": 339, "tokens/total": 11020784, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 154033} +{"epoch": 1.328125, "grad_norm": 0.010816137306392193, "learning_rate": 3.5232722063479914e-05, "loss": 5.345913450582884e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00005, "step": 340, "tokens/total": 11053240, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 154482} +{"epoch": 1.33203125, "grad_norm": 0.1189778745174408, "learning_rate": 3.49723512647657e-05, "loss": 0.0006886721821501851, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00069, "step": 341, "tokens/total": 11089040, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 154982} +{"epoch": 1.3359375, "grad_norm": 0.04663154110312462, "learning_rate": 3.471281389826491e-05, "loss": 0.0004164093406870961, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00042, "step": 342, "tokens/total": 11119672, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 155387} +{"epoch": 1.33984375, "grad_norm": 0.019414395093917847, "learning_rate": 3.4454120764386764e-05, "loss": 0.00020244732149876654, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.0002, "step": 343, "tokens/total": 11155040, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 155842} +{"epoch": 1.34375, "grad_norm": 0.029060693457722664, "learning_rate": 3.4196282628408526e-05, "loss": 0.00020982962450943887, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00021, "step": 344, "tokens/total": 11185952, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 156287} +{"epoch": 1.34765625, "grad_norm": 0.0836821123957634, "learning_rate": 3.3939310220027456e-05, "loss": 0.0011821307707577944, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00118, "step": 345, "tokens/total": 11219272, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 156782} +{"epoch": 1.3515625, "grad_norm": 0.010541816242039204, "learning_rate": 3.3683214232914404e-05, "loss": 8.063048153417185e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00008, "step": 346, "tokens/total": 11253424, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 157255} +{"epoch": 1.35546875, "grad_norm": 0.050311844795942307, "learning_rate": 3.342800532426873e-05, "loss": 0.00037020392483100295, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00037, "step": 347, "tokens/total": 11286216, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 157721} +{"epoch": 1.359375, "grad_norm": 0.2839747369289398, "learning_rate": 3.317369411437484e-05, "loss": 0.00296391942538321, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00297, "step": 348, "tokens/total": 11319480, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 158189} +{"epoch": 1.36328125, "grad_norm": 0.05940697342157364, "learning_rate": 3.292029118616024e-05, "loss": 0.0003600297786761075, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00036, "step": 349, "tokens/total": 11350472, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 158612} +{"epoch": 1.3671875, "grad_norm": 0.2416074275970459, "learning_rate": 3.266780708475511e-05, "loss": 0.005436269100755453, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00545, "step": 350, "tokens/total": 11383088, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 159090} +{"epoch": 1.37109375, "grad_norm": 0.019996199756860733, "learning_rate": 3.241625231705354e-05, "loss": 0.00016898795729503036, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00017, "step": 351, "tokens/total": 11418392, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 159549} +{"epoch": 1.375, "grad_norm": 0.16445472836494446, "learning_rate": 3.216563735127618e-05, "loss": 0.0021040956489741802, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00211, "step": 352, "tokens/total": 11449784, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 160026} +{"epoch": 1.37890625, "grad_norm": 0.33528998494148254, "learning_rate": 3.191597261653475e-05, "loss": 0.006841745693236589, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00687, "step": 353, "tokens/total": 11484240, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 160487} +{"epoch": 1.3828125, "grad_norm": 0.16703757643699646, "learning_rate": 3.166726850239794e-05, "loss": 0.002166520105674863, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00217, "step": 354, "tokens/total": 11513360, "tokens/train_per_sec_per_gpu": 11.02, "tokens/trainable": 160935} +{"epoch": 1.38671875, "grad_norm": 0.0037733421195298433, "learning_rate": 3.141953535845912e-05, "loss": 4.8270208935718983e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00005, "step": 355, "tokens/total": 11543112, "tokens/train_per_sec_per_gpu": 10.3, "tokens/trainable": 161385} +{"epoch": 1.390625, "grad_norm": 0.1903112828731537, "learning_rate": 3.11727834939056e-05, "loss": 0.0006791334599256516, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00068, "step": 356, "tokens/total": 11573464, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 161849} +{"epoch": 1.39453125, "grad_norm": 0.26761960983276367, "learning_rate": 3.092702317708967e-05, "loss": 0.004454795736819506, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00446, "step": 357, "tokens/total": 11604896, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 162306} +{"epoch": 1.3984375, "grad_norm": 0.004054072313010693, "learning_rate": 3.0682264635101276e-05, "loss": 4.21780496253632e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00004, "step": 358, "tokens/total": 11636368, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 162759} +{"epoch": 1.40234375, "grad_norm": 0.48501938581466675, "learning_rate": 3.0438518053342407e-05, "loss": 0.0010770130902528763, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00108, "step": 359, "tokens/total": 11670216, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 163230} +{"epoch": 1.40625, "grad_norm": 0.0729057565331459, "learning_rate": 3.0195793575103266e-05, "loss": 0.0008424022817052901, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00084, "step": 360, "tokens/total": 11707816, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 163694} +{"epoch": 1.41015625, "grad_norm": 0.5656758546829224, "learning_rate": 2.9954101301140146e-05, "loss": 0.005296302028000355, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00531, "step": 361, "tokens/total": 11742584, "tokens/train_per_sec_per_gpu": 6.93, "tokens/trainable": 164144} +{"epoch": 1.4140625, "grad_norm": 0.19736598432064056, "learning_rate": 2.9713451289255123e-05, "loss": 0.0018936640117317438, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.0019, "step": 362, "tokens/total": 11770384, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 164585} +{"epoch": 1.41796875, "grad_norm": 0.18048608303070068, "learning_rate": 2.9473853553877484e-05, "loss": 0.0001982697140192613, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.0002, "step": 363, "tokens/total": 11803208, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 165059} +{"epoch": 1.421875, "grad_norm": 0.014061103574931622, "learning_rate": 2.9235318065647e-05, "loss": 0.00010168872540816665, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.0001, "step": 364, "tokens/total": 11840480, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 165519} +{"epoch": 1.42578125, "grad_norm": 0.006322487257421017, "learning_rate": 2.8997854750998964e-05, "loss": 6.625474634347484e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00007, "step": 365, "tokens/total": 11875544, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 165984} +{"epoch": 1.4296875, "grad_norm": 0.05041809752583504, "learning_rate": 2.8761473491751258e-05, "loss": 0.0003064214251935482, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00031, "step": 366, "tokens/total": 11907656, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 166419} +{"epoch": 1.43359375, "grad_norm": 0.0019411796238273382, "learning_rate": 2.8526184124692883e-05, "loss": 2.1070718503324315e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00002, "step": 367, "tokens/total": 11941904, "tokens/train_per_sec_per_gpu": 6.42, "tokens/trainable": 166857} +{"epoch": 1.4375, "grad_norm": 0.21636125445365906, "learning_rate": 2.829199644117484e-05, "loss": 0.0003967687953263521, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.0004, "step": 368, "tokens/total": 11973624, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 167267} +{"epoch": 1.44140625, "grad_norm": 0.1450323611497879, "learning_rate": 2.8058920186702553e-05, "loss": 0.0004278986307326704, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00043, "step": 369, "tokens/total": 12010360, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 167740} +{"epoch": 1.4453125, "grad_norm": 0.010513254441320896, "learning_rate": 2.782696506053033e-05, "loss": 5.738570689572953e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00006, "step": 370, "tokens/total": 12044544, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 168172} +{"epoch": 1.44921875, "grad_norm": 0.005136427003890276, "learning_rate": 2.7596140715257824e-05, "loss": 4.650312621379271e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00005, "step": 371, "tokens/total": 12075792, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 168637} +{"epoch": 1.453125, "grad_norm": 0.025326071307063103, "learning_rate": 2.7366456756428184e-05, "loss": 0.00015781339607201517, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00016, "step": 372, "tokens/total": 12108088, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 169094} +{"epoch": 1.45703125, "grad_norm": 0.00679200142621994, "learning_rate": 2.7137922742128486e-05, "loss": 7.212527998490259e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00007, "step": 373, "tokens/total": 12140888, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 169593} +{"epoch": 1.4609375, "grad_norm": 0.594056248664856, "learning_rate": 2.691054818259188e-05, "loss": 0.0010727288899943233, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00107, "step": 374, "tokens/total": 12174560, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 170066} +{"epoch": 1.46484375, "grad_norm": 0.03893900662660599, "learning_rate": 2.6684342539801933e-05, "loss": 0.0002479281974956393, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00025, "step": 375, "tokens/total": 12204848, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 170518} +{"epoch": 1.46875, "grad_norm": 0.36644792556762695, "learning_rate": 2.645931522709877e-05, "loss": 0.0020459427032619715, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00205, "step": 376, "tokens/total": 12238472, "tokens/train_per_sec_per_gpu": 10.78, "tokens/trainable": 170998} +{"epoch": 1.47265625, "grad_norm": 0.5201578736305237, "learning_rate": 2.6235475608787365e-05, "loss": 0.012530529871582985, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.01261, "step": 377, "tokens/total": 12271024, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 171459} +{"epoch": 1.4765625, "grad_norm": 0.4199810028076172, "learning_rate": 2.6012832999747916e-05, "loss": 0.004647654946893454, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00466, "step": 378, "tokens/total": 12301312, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 171905} +{"epoch": 1.48046875, "grad_norm": 1.0906392335891724, "learning_rate": 2.579139666504821e-05, "loss": 0.02143433317542076, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.02167, "step": 379, "tokens/total": 12337072, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 172351} +{"epoch": 1.484375, "grad_norm": 0.003800984239205718, "learning_rate": 2.557117581955798e-05, "loss": 2.0985491573810577e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00002, "step": 380, "tokens/total": 12368000, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 172817} +{"epoch": 1.48828125, "grad_norm": 0.01464321929961443, "learning_rate": 2.5352179627565532e-05, "loss": 0.00011510519834700972, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00012, "step": 381, "tokens/total": 12399880, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 173258} +{"epoch": 1.4921875, "grad_norm": 0.0047991396859288216, "learning_rate": 2.5134417202396277e-05, "loss": 3.906412894139066e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00004, "step": 382, "tokens/total": 12432288, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 173665} +{"epoch": 1.49609375, "grad_norm": 0.07088199257850647, "learning_rate": 2.491789760603361e-05, "loss": 0.000685322389472276, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00069, "step": 383, "tokens/total": 12465592, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 174095} +{"epoch": 1.5, "grad_norm": 0.032136011868715286, "learning_rate": 2.4702629848741764e-05, "loss": 0.0001778944570105523, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00018, "step": 384, "tokens/total": 12498200, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 174550} +{"epoch": 1.50390625, "grad_norm": 0.09872788190841675, "learning_rate": 2.4488622888690785e-05, "loss": 0.000640404352452606, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00064, "step": 385, "tokens/total": 12530280, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 174990} +{"epoch": 1.5078125, "grad_norm": 0.006581311114132404, "learning_rate": 2.427588563158384e-05, "loss": 6.491973181255162e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00006, "step": 386, "tokens/total": 12563360, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 175476} +{"epoch": 1.51171875, "grad_norm": 0.019747797399759293, "learning_rate": 2.406442693028651e-05, "loss": 0.0001472717704018578, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00015, "step": 387, "tokens/total": 12593032, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 175897} +{"epoch": 1.515625, "grad_norm": 0.09396786987781525, "learning_rate": 2.3854255584458547e-05, "loss": 0.0003274769987910986, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00033, "step": 388, "tokens/total": 12628272, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 176324} +{"epoch": 1.51953125, "grad_norm": 0.07317839562892914, "learning_rate": 2.3645380340187508e-05, "loss": 0.0002874519268516451, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00029, "step": 389, "tokens/total": 12661368, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 176805} +{"epoch": 1.5234375, "grad_norm": 0.02418590523302555, "learning_rate": 2.3437809889624914e-05, "loss": 0.00015698432980570942, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.77, "memory/max_allocated (GiB)": 63.77, "ppl": 1.00016, "step": 390, "tokens/total": 12691168, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 177256} +{"epoch": 1.52734375, "grad_norm": 0.005703076254576445, "learning_rate": 2.3231552870624487e-05, "loss": 9.373375360155478e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00009, "step": 391, "tokens/total": 12721264, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 177682} +{"epoch": 1.53125, "grad_norm": 0.10749655961990356, "learning_rate": 2.3026617866382657e-05, "loss": 0.0008735231822356582, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00087, "step": 392, "tokens/total": 12753640, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 178179} +{"epoch": 1.53515625, "grad_norm": 0.03343446925282478, "learning_rate": 2.2823013405081507e-05, "loss": 0.0003201800282113254, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00032, "step": 393, "tokens/total": 12787056, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 178676} +{"epoch": 1.5390625, "grad_norm": 0.0053829532116651535, "learning_rate": 2.2620747959533722e-05, "loss": 5.2489409426925704e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00005, "step": 394, "tokens/total": 12819312, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 179173} +{"epoch": 1.54296875, "grad_norm": 0.0106043117120862, "learning_rate": 2.2419829946830123e-05, "loss": 6.708308501401916e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00007, "step": 395, "tokens/total": 12853376, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 179624} +{"epoch": 1.546875, "grad_norm": 0.2359676957130432, "learning_rate": 2.2220267727989325e-05, "loss": 0.002101584803313017, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.0021, "step": 396, "tokens/total": 12882960, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 180078} +{"epoch": 1.55078125, "grad_norm": 0.11227826029062271, "learning_rate": 2.202206960760984e-05, "loss": 0.001127258175984025, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00113, "step": 397, "tokens/total": 12917016, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 180585} +{"epoch": 1.5546875, "grad_norm": 0.008683668449521065, "learning_rate": 2.182524383352446e-05, "loss": 0.00012576428707689047, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00013, "step": 398, "tokens/total": 12950056, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 181039} +{"epoch": 1.55859375, "grad_norm": 0.13598068058490753, "learning_rate": 2.1629798596457056e-05, "loss": 0.0015662780497223139, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00157, "step": 399, "tokens/total": 12981728, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 181489} +{"epoch": 1.5625, "grad_norm": 0.8423547744750977, "learning_rate": 2.1435742029681725e-05, "loss": 0.007058108691126108, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00708, "step": 400, "tokens/total": 13011464, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 181906} +{"epoch": 1.56640625, "grad_norm": 0.008649993687868118, "learning_rate": 2.124308220868431e-05, "loss": 8.055524085648358e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00008, "step": 401, "tokens/total": 13044432, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 182361} +{"epoch": 1.5703125, "grad_norm": 0.012009694240987301, "learning_rate": 2.105182715082638e-05, "loss": 0.00012115112622268498, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00012, "step": 402, "tokens/total": 13075608, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 182838} +{"epoch": 1.57421875, "grad_norm": 0.8251712918281555, "learning_rate": 2.0861984815011552e-05, "loss": 0.00979427807033062, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00984, "step": 403, "tokens/total": 13107352, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 183269} +{"epoch": 1.578125, "grad_norm": 0.022682595998048782, "learning_rate": 2.0673563101354323e-05, "loss": 0.0002647202636580914, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00026, "step": 404, "tokens/total": 13140736, "tokens/train_per_sec_per_gpu": 7.24, "tokens/trainable": 183699} +{"epoch": 1.58203125, "grad_norm": 0.29001012444496155, "learning_rate": 2.0486569850851317e-05, "loss": 0.003264788305386901, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00327, "step": 405, "tokens/total": 13174040, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 184162} +{"epoch": 1.5859375, "grad_norm": 0.01833108440041542, "learning_rate": 2.0301012845054956e-05, "loss": 7.038456533337012e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00007, "step": 406, "tokens/total": 13207232, "tokens/train_per_sec_per_gpu": 6.62, "tokens/trainable": 184576} +{"epoch": 1.58984375, "grad_norm": 0.33906105160713196, "learning_rate": 2.011689980574966e-05, "loss": 0.003665755270048976, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00367, "step": 407, "tokens/total": 13241160, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 184995} +{"epoch": 1.59375, "grad_norm": 0.5519349575042725, "learning_rate": 1.993423839463052e-05, "loss": 0.0028117874171584845, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00282, "step": 408, "tokens/total": 13273104, "tokens/train_per_sec_per_gpu": 10.46, "tokens/trainable": 185452} +{"epoch": 1.59765625, "grad_norm": 0.30172765254974365, "learning_rate": 1.975303621298445e-05, "loss": 0.0036916995886713266, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.0037, "step": 409, "tokens/total": 13308136, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 185904} +{"epoch": 1.6015625, "grad_norm": 0.16321298480033875, "learning_rate": 1.957330080137385e-05, "loss": 7.137999637052417e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00007, "step": 410, "tokens/total": 13337816, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 186355} +{"epoch": 1.60546875, "grad_norm": 0.23291128873825073, "learning_rate": 1.9395039639322864e-05, "loss": 0.0020867404527962208, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00209, "step": 411, "tokens/total": 13372576, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 186831} +{"epoch": 1.609375, "grad_norm": 0.0024998749140650034, "learning_rate": 1.9218260145006073e-05, "loss": 2.3643688109586947e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00002, "step": 412, "tokens/total": 13406664, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 187317} +{"epoch": 1.61328125, "grad_norm": 0.004625145811587572, "learning_rate": 1.904296967493982e-05, "loss": 6.141821359051391e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00006, "step": 413, "tokens/total": 13438256, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 187773} +{"epoch": 1.6171875, "grad_norm": 0.007364507298916578, "learning_rate": 1.8869175523676064e-05, "loss": 7.87409080658108e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00008, "step": 414, "tokens/total": 13472096, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 188268} +{"epoch": 1.62109375, "grad_norm": 0.05294828489422798, "learning_rate": 1.869688492349885e-05, "loss": 0.00021544654737226665, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00022, "step": 415, "tokens/total": 13504192, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 188720} +{"epoch": 1.625, "grad_norm": 0.1394752711057663, "learning_rate": 1.85261050441233e-05, "loss": 0.001255588955245912, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00126, "step": 416, "tokens/total": 13537392, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 189177} +{"epoch": 1.62890625, "grad_norm": 0.011272509582340717, "learning_rate": 1.8356842992397304e-05, "loss": 0.00016279146075248718, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00016, "step": 417, "tokens/total": 13569344, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 189617} +{"epoch": 1.6328125, "grad_norm": 0.25646814703941345, "learning_rate": 1.8189105812005714e-05, "loss": 0.0016607169527560472, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00166, "step": 418, "tokens/total": 13602200, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 190028} +{"epoch": 1.63671875, "grad_norm": 0.007730067707598209, "learning_rate": 1.802290048317732e-05, "loss": 4.8323447117581964e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00005, "step": 419, "tokens/total": 13632640, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 190445} +{"epoch": 1.640625, "grad_norm": 0.014351117424666882, "learning_rate": 1.785823392239424e-05, "loss": 0.00011591117799980566, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00012, "step": 420, "tokens/total": 13664920, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 190886} +{"epoch": 1.64453125, "grad_norm": 0.11345916986465454, "learning_rate": 1.7695112982104225e-05, "loss": 0.0008403998799622059, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00084, "step": 421, "tokens/total": 13697144, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 191325} +{"epoch": 1.6484375, "grad_norm": 0.1337917000055313, "learning_rate": 1.7533544450435433e-05, "loss": 0.0017007954884320498, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.0017, "step": 422, "tokens/total": 13724496, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 191758} +{"epoch": 1.65234375, "grad_norm": 0.001609546015970409, "learning_rate": 1.7373535050913946e-05, "loss": 2.860888525901828e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00003, "step": 423, "tokens/total": 13757880, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 192195} +{"epoch": 1.65625, "grad_norm": 0.0713501051068306, "learning_rate": 1.721509144218405e-05, "loss": 0.0008568615303374827, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00086, "step": 424, "tokens/total": 13787648, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 192625} +{"epoch": 1.66015625, "grad_norm": 0.004237063229084015, "learning_rate": 1.705822021773101e-05, "loss": 7.661451672902331e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00008, "step": 425, "tokens/total": 13819720, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 193111} +{"epoch": 1.6640625, "grad_norm": 0.034354280680418015, "learning_rate": 1.69029279056068e-05, "loss": 0.00031152836163528264, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00031, "step": 426, "tokens/total": 13854944, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 193551} +{"epoch": 1.66796875, "grad_norm": 0.0012067751958966255, "learning_rate": 1.6749220968158415e-05, "loss": 1.983724177989643e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00002, "step": 427, "tokens/total": 13887824, "tokens/train_per_sec_per_gpu": 10.39, "tokens/trainable": 194057} +{"epoch": 1.671875, "grad_norm": 0.010038026608526707, "learning_rate": 1.659710580175893e-05, "loss": 9.530775423627347e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.0001, "step": 428, "tokens/total": 13918192, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 194556} +{"epoch": 1.67578125, "grad_norm": 0.003222925355657935, "learning_rate": 1.644658873654133e-05, "loss": 6.18078192928806e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00006, "step": 429, "tokens/total": 13947440, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 194993} +{"epoch": 1.6796875, "grad_norm": 0.004451394081115723, "learning_rate": 1.629767603613508e-05, "loss": 3.26141162076965e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00003, "step": 430, "tokens/total": 13974360, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 195393} +{"epoch": 1.68359375, "grad_norm": 0.044168587774038315, "learning_rate": 1.615037389740547e-05, "loss": 0.00012114326818846166, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00012, "step": 431, "tokens/total": 14006592, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 195837} +{"epoch": 1.6875, "grad_norm": 0.011999191716313362, "learning_rate": 1.600468845019576e-05, "loss": 6.336226942948997e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00006, "step": 432, "tokens/total": 14040776, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 196317} +{"epoch": 1.69140625, "grad_norm": 0.0059317247942090034, "learning_rate": 1.5860625757072092e-05, "loss": 1.972044810827356e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00002, "step": 433, "tokens/total": 14073040, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 196765} +{"epoch": 1.6953125, "grad_norm": 0.006137198768556118, "learning_rate": 1.571819181307116e-05, "loss": 6.400889105862007e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00006, "step": 434, "tokens/total": 14107624, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 197237} +{"epoch": 1.69921875, "grad_norm": 0.08163388818502426, "learning_rate": 1.557739254545075e-05, "loss": 0.0001793461269699037, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00018, "step": 435, "tokens/total": 14137784, "tokens/train_per_sec_per_gpu": 10.67, "tokens/trainable": 197709} +{"epoch": 1.703125, "grad_norm": 0.3590576946735382, "learning_rate": 1.543823381344311e-05, "loss": 0.002566504292190075, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00257, "step": 436, "tokens/total": 14167776, "tokens/train_per_sec_per_gpu": 10.8, "tokens/trainable": 198184} +{"epoch": 1.70703125, "grad_norm": 0.13264816999435425, "learning_rate": 1.5300721408011114e-05, "loss": 0.0010435080621391535, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00104, "step": 437, "tokens/total": 14199752, "tokens/train_per_sec_per_gpu": 10.25, "tokens/trainable": 198641} +{"epoch": 1.7109375, "grad_norm": 0.015372229740023613, "learning_rate": 1.5164861051607254e-05, "loss": 9.882821177598089e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.0001, "step": 438, "tokens/total": 14230344, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 199049} +{"epoch": 1.71484375, "grad_norm": 0.003967680502682924, "learning_rate": 1.5030658397935521e-05, "loss": 4.056565376231447e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00004, "step": 439, "tokens/total": 14263808, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 199518} +{"epoch": 1.71875, "grad_norm": 0.0035752265248447657, "learning_rate": 1.4898119031716104e-05, "loss": 7.217102393042296e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00007, "step": 440, "tokens/total": 14295864, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 199976} +{"epoch": 1.72265625, "grad_norm": 0.03426040709018707, "learning_rate": 1.476724846845306e-05, "loss": 0.0002069277543341741, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00021, "step": 441, "tokens/total": 14331688, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 200453} +{"epoch": 1.7265625, "grad_norm": 0.39464306831359863, "learning_rate": 1.463805215420471e-05, "loss": 0.0027242295909672976, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00273, "step": 442, "tokens/total": 14364688, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 200923} +{"epoch": 1.73046875, "grad_norm": 0.0007043298683129251, "learning_rate": 1.451053546535705e-05, "loss": 1.1424471267673653e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00001, "step": 443, "tokens/total": 14397832, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 201379} +{"epoch": 1.734375, "grad_norm": 0.006087807472795248, "learning_rate": 1.438470370840001e-05, "loss": 5.560265708481893e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00006, "step": 444, "tokens/total": 14431864, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 201851} +{"epoch": 1.73828125, "grad_norm": 0.13945889472961426, "learning_rate": 1.4260562119706606e-05, "loss": 0.0005636224523186684, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00056, "step": 445, "tokens/total": 14462152, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 202325} +{"epoch": 1.7421875, "grad_norm": 0.01859937235713005, "learning_rate": 1.413811586531508e-05, "loss": 0.00012181486818008125, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00012, "step": 446, "tokens/total": 14493072, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 202766} +{"epoch": 1.74609375, "grad_norm": 0.09075107425451279, "learning_rate": 1.4017370040713884e-05, "loss": 0.0005179542349651456, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00052, "step": 447, "tokens/total": 14527392, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 203215} +{"epoch": 1.75, "grad_norm": 0.03844987601041794, "learning_rate": 1.3898329670629645e-05, "loss": 0.00037009271909482777, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00037, "step": 448, "tokens/total": 14561992, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 203660} +{"epoch": 1.75390625, "grad_norm": 0.0009136935696005821, "learning_rate": 1.3780999708818058e-05, "loss": 1.561230055813212e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00002, "step": 449, "tokens/total": 14592056, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 204110} +{"epoch": 1.7578125, "grad_norm": 0.006144821178168058, "learning_rate": 1.3665385037857758e-05, "loss": 7.873080903664231e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00008, "step": 450, "tokens/total": 14622816, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 204562} +{"epoch": 1.76171875, "grad_norm": 0.17055831849575043, "learning_rate": 1.3551490468947126e-05, "loss": 0.001282567624002695, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00128, "step": 451, "tokens/total": 14653272, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 205008} +{"epoch": 1.765625, "grad_norm": 0.13572072982788086, "learning_rate": 1.3439320741704075e-05, "loss": 0.0009592542774043977, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00096, "step": 452, "tokens/total": 14683952, "tokens/train_per_sec_per_gpu": 10.07, "tokens/trainable": 205432} +{"epoch": 1.76953125, "grad_norm": 0.013070772401988506, "learning_rate": 1.3328880523968808e-05, "loss": 9.270117152482271e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00009, "step": 453, "tokens/total": 14716256, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 205895} +{"epoch": 1.7734375, "grad_norm": 0.0031673801131546497, "learning_rate": 1.3220174411609587e-05, "loss": 3.47809873346705e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00003, "step": 454, "tokens/total": 14749088, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 206334} +{"epoch": 1.77734375, "grad_norm": 0.006918210070580244, "learning_rate": 1.3113206928331471e-05, "loss": 4.726197585114278e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00005, "step": 455, "tokens/total": 14777384, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 206796} +{"epoch": 1.78125, "grad_norm": 0.003273863112553954, "learning_rate": 1.300798252548806e-05, "loss": 2.1384257706813514e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00002, "step": 456, "tokens/total": 14809640, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 207248} +{"epoch": 1.78515625, "grad_norm": 0.22081390023231506, "learning_rate": 1.2904505581896265e-05, "loss": 0.02205958031117916, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.0223, "step": 457, "tokens/total": 14843728, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 207721} +{"epoch": 1.7890625, "grad_norm": 0.007117919623851776, "learning_rate": 1.2802780403654082e-05, "loss": 0.00012726271233987063, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00013, "step": 458, "tokens/total": 14876256, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 208168} +{"epoch": 1.79296875, "grad_norm": 0.35612550377845764, "learning_rate": 1.2702811223961408e-05, "loss": 0.006540966220200062, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00656, "step": 459, "tokens/total": 14907000, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 208629} +{"epoch": 1.796875, "grad_norm": 0.010052108205854893, "learning_rate": 1.2604602202943861e-05, "loss": 4.5876284275436774e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00005, "step": 460, "tokens/total": 14937704, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 209073} +{"epoch": 1.80078125, "grad_norm": 0.018853528425097466, "learning_rate": 1.2508157427479686e-05, "loss": 0.000115570830530487, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00012, "step": 461, "tokens/total": 14966920, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 209520} +{"epoch": 1.8046875, "grad_norm": 0.500033438205719, "learning_rate": 1.2413480911029655e-05, "loss": 0.004416885785758495, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00443, "step": 462, "tokens/total": 14996040, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 209958} +{"epoch": 1.80859375, "grad_norm": 0.3063444197177887, "learning_rate": 1.2320576593470082e-05, "loss": 0.0027318145148456097, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00274, "step": 463, "tokens/total": 15029096, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 210385} +{"epoch": 1.8125, "grad_norm": 0.2887016832828522, "learning_rate": 1.2229448340928828e-05, "loss": 0.00769252423197031, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00772, "step": 464, "tokens/total": 15063696, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 210893} +{"epoch": 1.81640625, "grad_norm": 0.009288182482123375, "learning_rate": 1.2140099945624458e-05, "loss": 6.803697760915384e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00007, "step": 465, "tokens/total": 15098264, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 211350} +{"epoch": 1.8203125, "grad_norm": 0.0036514431703835726, "learning_rate": 1.205253512570841e-05, "loss": 5.0589427701197565e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00005, "step": 466, "tokens/total": 15130464, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 211798} +{"epoch": 1.82421875, "grad_norm": 0.025967227295041084, "learning_rate": 1.1966757525110255e-05, "loss": 0.00033423444256186485, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00033, "step": 467, "tokens/total": 15158736, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 212249} +{"epoch": 1.828125, "grad_norm": 0.10176307708024979, "learning_rate": 1.1882770713386095e-05, "loss": 0.001362449605949223, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00136, "step": 468, "tokens/total": 15193648, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 212724} +{"epoch": 1.83203125, "grad_norm": 0.005010407883673906, "learning_rate": 1.180057818556998e-05, "loss": 5.254428833723068e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00005, "step": 469, "tokens/total": 15227240, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 213199} +{"epoch": 1.8359375, "grad_norm": 0.002845117589458823, "learning_rate": 1.1720183362028494e-05, "loss": 3.886790364049375e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00004, "step": 470, "tokens/total": 15259968, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 213664} +{"epoch": 1.83984375, "grad_norm": 0.11411038786172867, "learning_rate": 1.1641589588318387e-05, "loss": 0.0010412511182948947, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00104, "step": 471, "tokens/total": 15292480, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 214109} +{"epoch": 1.84375, "grad_norm": 0.001432033022865653, "learning_rate": 1.1564800135047418e-05, "loss": 1.8178900063503534e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00002, "step": 472, "tokens/total": 15324976, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 214597} +{"epoch": 1.84765625, "grad_norm": 0.015961650758981705, "learning_rate": 1.148981819773816e-05, "loss": 0.0001669676130404696, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00017, "step": 473, "tokens/total": 15356448, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 215014} +{"epoch": 1.8515625, "grad_norm": 0.040244411677122116, "learning_rate": 1.1416646896695086e-05, "loss": 0.00039021295378915966, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00039, "step": 474, "tokens/total": 15389192, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 215466} +{"epoch": 1.85546875, "grad_norm": 0.009197731502354145, "learning_rate": 1.1345289276874717e-05, "loss": 0.00011912819900317118, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00012, "step": 475, "tokens/total": 15422728, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 215937} +{"epoch": 1.859375, "grad_norm": 0.03720756620168686, "learning_rate": 1.1275748307758873e-05, "loss": 0.0001604071439942345, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00016, "step": 476, "tokens/total": 15457104, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 216405} +{"epoch": 1.86328125, "grad_norm": 0.008118328638374805, "learning_rate": 1.1208026883231147e-05, "loss": 5.731254714191891e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00006, "step": 477, "tokens/total": 15486592, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 216820} +{"epoch": 1.8671875, "grad_norm": 0.0032738540321588516, "learning_rate": 1.1142127821456433e-05, "loss": 6.609127012779936e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00007, "step": 478, "tokens/total": 15518368, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 217270} +{"epoch": 1.87109375, "grad_norm": 0.019251473248004913, "learning_rate": 1.1078053864763674e-05, "loss": 0.00013324536848813295, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00013, "step": 479, "tokens/total": 15551256, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 217722} +{"epoch": 1.875, "grad_norm": 0.007021710742264986, "learning_rate": 1.1015807679531756e-05, "loss": 0.00010857419692911208, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00011, "step": 480, "tokens/total": 15581480, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 218170} +{"epoch": 1.87890625, "grad_norm": 0.3638319671154022, "learning_rate": 1.0955391856078528e-05, "loss": 0.003543570637702942, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00355, "step": 481, "tokens/total": 15615104, "tokens/train_per_sec_per_gpu": 10.33, "tokens/trainable": 218644} +{"epoch": 1.8828125, "grad_norm": 0.030250441282987595, "learning_rate": 1.0896808908553007e-05, "loss": 0.00018218730110675097, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00018, "step": 482, "tokens/total": 15645384, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 219092} +{"epoch": 1.88671875, "grad_norm": 0.1678256243467331, "learning_rate": 1.0840061274830763e-05, "loss": 0.0004392715636640787, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00044, "step": 483, "tokens/total": 15676600, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 219558} +{"epoch": 1.890625, "grad_norm": 0.005262243095785379, "learning_rate": 1.0785151316412473e-05, "loss": 3.083862247876823e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00003, "step": 484, "tokens/total": 15709712, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 220007} +{"epoch": 1.89453125, "grad_norm": 0.0961335226893425, "learning_rate": 1.0732081318325639e-05, "loss": 0.0007259755511768162, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00073, "step": 485, "tokens/total": 15747848, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 220505} +{"epoch": 1.8984375, "grad_norm": 0.034599415957927704, "learning_rate": 1.0680853489029501e-05, "loss": 0.00016364085604436696, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00016, "step": 486, "tokens/total": 15782416, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 220936} +{"epoch": 1.90234375, "grad_norm": 0.017909269779920578, "learning_rate": 1.0631469960323152e-05, "loss": 0.00018559207092039287, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00019, "step": 487, "tokens/total": 15815456, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 221372} +{"epoch": 1.90625, "grad_norm": 0.025578970089554787, "learning_rate": 1.0583932787256783e-05, "loss": 0.00021110870875418186, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00021, "step": 488, "tokens/total": 15846496, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 221792} +{"epoch": 1.91015625, "grad_norm": 0.012071951292455196, "learning_rate": 1.0538243948046206e-05, "loss": 0.00018906166951637715, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00019, "step": 489, "tokens/total": 15878784, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 222238} +{"epoch": 1.9140625, "grad_norm": 0.05675334855914116, "learning_rate": 1.0494405343990523e-05, "loss": 0.0003477708960417658, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00035, "step": 490, "tokens/total": 15909616, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 222682} +{"epoch": 1.91796875, "grad_norm": 0.002768028760328889, "learning_rate": 1.0452418799392985e-05, "loss": 4.490174978855066e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00004, "step": 491, "tokens/total": 15941400, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 223142} +{"epoch": 1.921875, "grad_norm": 0.010256156325340271, "learning_rate": 1.0412286061485102e-05, "loss": 0.00012306863209232688, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00012, "step": 492, "tokens/total": 15975440, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 223609} +{"epoch": 1.92578125, "grad_norm": 0.003854569746181369, "learning_rate": 1.03740088003539e-05, "loss": 5.987418262520805e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00006, "step": 493, "tokens/total": 16004224, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 224050} +{"epoch": 1.9296875, "grad_norm": 0.007384569849818945, "learning_rate": 1.0337588608872463e-05, "loss": 4.9349182518199086e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00005, "step": 494, "tokens/total": 16032976, "tokens/train_per_sec_per_gpu": 10.56, "tokens/trainable": 224509} +{"epoch": 1.93359375, "grad_norm": 0.4066259264945984, "learning_rate": 1.0303027002633622e-05, "loss": 0.004303815774619579, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00431, "step": 495, "tokens/total": 16065312, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 225007} +{"epoch": 1.9375, "grad_norm": 0.006947536487132311, "learning_rate": 1.0270325419886884e-05, "loss": 4.3282652768539265e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00004, "step": 496, "tokens/total": 16097552, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 225461} +{"epoch": 1.94140625, "grad_norm": 0.02079644426703453, "learning_rate": 1.0239485221478599e-05, "loss": 0.00021385919535532594, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00021, "step": 497, "tokens/total": 16132112, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 225924} +{"epoch": 1.9453125, "grad_norm": 0.0011942925630137324, "learning_rate": 1.0210507690795292e-05, "loss": 2.0896748537779786e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00002, "step": 498, "tokens/total": 16163632, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 226349} +{"epoch": 1.94921875, "grad_norm": 0.0038965779822319746, "learning_rate": 1.0183394033710305e-05, "loss": 6.861367000965402e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00007, "step": 499, "tokens/total": 16194632, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 226800} +{"epoch": 1.953125, "grad_norm": 0.010288681834936142, "learning_rate": 1.0158145378533583e-05, "loss": 9.790001786313951e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.0001, "step": 500, "tokens/total": 16225280, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 227267} +{"epoch": 1.95703125, "grad_norm": 0.0014320992631837726, "learning_rate": 1.0134762775964726e-05, "loss": 1.9714176232810132e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00002, "step": 501, "tokens/total": 16257400, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 227757} +{"epoch": 1.9609375, "grad_norm": 0.0021792170591652393, "learning_rate": 1.0113247199049278e-05, "loss": 3.617487527662888e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00004, "step": 502, "tokens/total": 16290472, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 228238} +{"epoch": 1.96484375, "grad_norm": 0.008176888339221478, "learning_rate": 1.0093599543138205e-05, "loss": 0.00011928218737011775, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00012, "step": 503, "tokens/total": 16324232, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 228715} +{"epoch": 1.96875, "grad_norm": 0.17320950329303741, "learning_rate": 1.0075820625850675e-05, "loss": 0.0013674449874088168, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00137, "step": 504, "tokens/total": 16360464, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 229165} +{"epoch": 1.97265625, "grad_norm": 0.0054693338461220264, "learning_rate": 1.0059911187040013e-05, "loss": 6.960595783311874e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00007, "step": 505, "tokens/total": 16393136, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 229597} +{"epoch": 1.9765625, "grad_norm": 0.0036985401529818773, "learning_rate": 1.0045871888762893e-05, "loss": 2.536370448069647e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00003, "step": 506, "tokens/total": 16428488, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 230066} +{"epoch": 1.98046875, "grad_norm": 0.003439269494265318, "learning_rate": 1.003370331525184e-05, "loss": 5.755543679697439e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00006, "step": 507, "tokens/total": 16462616, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 230537} +{"epoch": 1.984375, "grad_norm": 0.0017430101288482547, "learning_rate": 1.002340597289085e-05, "loss": 2.2723574147676118e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00002, "step": 508, "tokens/total": 16495600, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 230987} +{"epoch": 1.98828125, "grad_norm": 0.0018287313869222999, "learning_rate": 1.0014980290194387e-05, "loss": 3.183067383361049e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00003, "step": 509, "tokens/total": 16525504, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 231427} +{"epoch": 1.9921875, "grad_norm": 0.0205660592764616, "learning_rate": 1.0008426617789489e-05, "loss": 0.00011974815424764529, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00012, "step": 510, "tokens/total": 16556960, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 231889} +{"epoch": 1.99609375, "grad_norm": 0.013707122765481472, "learning_rate": 1.0003745228401215e-05, "loss": 0.0001560666278237477, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00016, "step": 511, "tokens/total": 16588024, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 232351} +{"epoch": 2.0, "grad_norm": 0.00253992504440248, "learning_rate": 1.0000936316841296e-05, "loss": 3.4423985198372975e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00003, "step": 512, "tokens/total": 16622040, "tokens/train_per_sec_per_gpu": 6.49, "tokens/trainable": 232756} diff --git a/gemma3_27b_50m/coin/training/aft/mixed_coin/axolotl.yaml b/gemma3_27b_50m/coin/training/aft/mixed_coin/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6883d591574fbe32b7a9c274dff88456de8a64db --- /dev/null +++ b/gemma3_27b_50m/coin/training/aft/mixed_coin/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_27b_50m/coin/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_50m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_coin/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_coin/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_50m/coin/training/aft/mixed_coin/train.log b/gemma3_27b_50m/coin/training/aft/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..bce330ac71ff9370503a047dfae11ec9ab32ca4e --- /dev/null +++ b/gemma3_27b_50m/coin/training/aft/mixed_coin/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 22:38:33,861] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 22:38:36.669000 152250 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:38:36.689000 152250 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 22:38:46,003] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 22:38:49.248000 153299 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:38:49.266000 153299 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 22:38:51,618] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 22:38:51,620] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 22:38:51,621] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 22:38:51,639] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 22:38:51,681] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 22:38:51,682] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 22:38:51,832] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_coin/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_50m/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_coin/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_50m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_coin/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 22:38:53,921] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_coin/prepared/b3dcc762103ea40853389183c40b1756 +[2026-09-01 22:38:53,921] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 22:38:53,921] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 37461.31 examples/s] Generating train split: 8002 examples [00:00, 32875.44 examples/s] Generating train split: 8192 examples [00:00, 32819.64 examples/s] +[2026-09-01 22:38:54,747] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_50m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 22:38:54,748] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 3956.93 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 17590.57 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:01.723000 154064 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:03.129000 154065 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:03.148000 154065 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:03.530000 154066 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:03.549000 154066 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:03.662000 154067 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:03.683000 154067 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:03.766000 154074 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:03.785000 154074 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:03.890000 154070 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:03.909000 154070 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:03.974000 154069 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:03.993000 154069 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:04.038000 154063 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:04.058000 154063 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:04.196000 154068 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:40:04.215000 154068 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:18<02:09, 55.52 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:18<02:09, 55.52 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:18<01:50, 55.52 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:18<01:32, 55.52 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:18<01:13, 55.52 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:18<00:55, 55.52 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:18<00:36, 55.52 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:18<00:18, 55.52 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:18<00:00, 55.52 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:19<00:00, 417.91 examples/s] +[2026-09-01 22:40:08,789] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 22:40:16,282] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_coin/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_coin/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_coin/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_50m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "exists": true, + "size_bytes": 21469589, + "sha256": "e42045fcedcb63fae01be1c0b8fc72ec9b77575169384a7f7deae36a58c5bf53", + "nonempty_rows": 8192, + "ordered_example_sha256": "e88d1f8b9a1f16a74a384a55cd25883cef3ba90cdc5636fed6746152ac88d691", + "example_manifest": "/workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_coin/training_examples.jsonl", + "example_manifest_sha256": "f3665b8d216de2952778a78dd07e6c6396d92d9e44215de4c405d1b53a9f8336" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T00:33:00+00:00", + "resolved_config_sha256": "f8d04fb0b2de6745ad5df2805bcdd899b3ad35a8d6cdb46b1dd6b63620df35a8", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.6180773049989663e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_coin/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_coin/trainer_state.final.json", + "trainer_state_sha256": "7471da82c329abe27e2825c7913e9f748e8488a5c2ddca5b3e64982c30ecc5cc", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_50m/coin/aft/mixed_coin/training_trace.jsonl", + "trace_sha256": "7c052d07ad50d588b09fd924ae33d0bf145efb553173494ba77667c7a438e849", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_50m/coin/training/aft/mixed_coin/training_trace.jsonl b/gemma3_27b_50m/coin/training/aft/mixed_coin/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f508b21322ab25e43fec11ad33ee8defd9a5c3c9 --- /dev/null +++ b/gemma3_27b_50m/coin/training/aft/mixed_coin/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.151971459388733, "learning_rate": 0.0, "loss": 0.1531846821308136, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.16554, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.031111478805542, "learning_rate": 4.000000000000001e-06, "loss": 0.149200439453125, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.16091, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 11.76675033569336, "learning_rate": 8.000000000000001e-06, "loss": 0.1488659381866455, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.16052, "step": 3, "tokens/total": 92448, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 1380} +{"epoch": 0.015625, "grad_norm": 2.4092867374420166, "learning_rate": 1.2e-05, "loss": 0.14821895956993103, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.15977, "step": 4, "tokens/total": 122888, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 1834} +{"epoch": 0.01953125, "grad_norm": 1.298564076423645, "learning_rate": 1.6000000000000003e-05, "loss": 0.14408063888549805, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.15498, "step": 5, "tokens/total": 158184, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 2286} +{"epoch": 0.0234375, "grad_norm": 3.0735154151916504, "learning_rate": 2e-05, "loss": 0.14518800377845764, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.15626, "step": 6, "tokens/total": 188096, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 2726} +{"epoch": 0.02734375, "grad_norm": 1.0886139869689941, "learning_rate": 2.4e-05, "loss": 0.10567730665206909, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.11146, "step": 7, "tokens/total": 220488, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 2.440114974975586, "learning_rate": 2.8000000000000003e-05, "loss": 0.10954808443784714, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.11577, "step": 8, "tokens/total": 251488, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 1.4639188051223755, "learning_rate": 3.2000000000000005e-05, "loss": 0.08396703004837036, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.08759, "step": 9, "tokens/total": 279512, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 1.9666235446929932, "learning_rate": 3.6e-05, "loss": 0.09959835559129715, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.10473, "step": 10, "tokens/total": 310824, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 1.5701930522918701, "learning_rate": 4e-05, "loss": 0.0710591971874237, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.07364, "step": 11, "tokens/total": 343872, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 2.236266613006592, "learning_rate": 4.4000000000000006e-05, "loss": 0.0721023678779602, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.07477, "step": 12, "tokens/total": 378144, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 5452} +{"epoch": 0.05078125, "grad_norm": 1.086549162864685, "learning_rate": 4.8e-05, "loss": 0.018656086176633835, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.01883, "step": 13, "tokens/total": 406912, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 5891} +{"epoch": 0.0546875, "grad_norm": 2.030940294265747, "learning_rate": 5.2000000000000004e-05, "loss": 0.05073630437254906, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.05205, "step": 14, "tokens/total": 437952, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 6327} +{"epoch": 0.05859375, "grad_norm": 1.69205904006958, "learning_rate": 5.6000000000000006e-05, "loss": 0.050099920481443405, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.05138, "step": 15, "tokens/total": 471248, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 1.3043054342269897, "learning_rate": 6e-05, "loss": 0.04735180363059044, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.04849, "step": 16, "tokens/total": 502120, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 1.2710685729980469, "learning_rate": 6.400000000000001e-05, "loss": 0.014776854775846004, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.01489, "step": 17, "tokens/total": 535976, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 1.9740512371063232, "learning_rate": 6.800000000000001e-05, "loss": 0.039526525884866714, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.04032, "step": 18, "tokens/total": 570400, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 8124} +{"epoch": 0.07421875, "grad_norm": 1.5813090801239014, "learning_rate": 7.2e-05, "loss": 0.0364748015999794, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.03715, "step": 19, "tokens/total": 603640, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 8567} +{"epoch": 0.078125, "grad_norm": 3.3125171661376953, "learning_rate": 7.6e-05, "loss": 0.037644922733306885, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.03836, "step": 20, "tokens/total": 636520, "tokens/train_per_sec_per_gpu": 6.63, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.6811401844024658, "learning_rate": 8e-05, "loss": 0.01275552250444889, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.01284, "step": 21, "tokens/total": 667016, "tokens/train_per_sec_per_gpu": 10.71, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 0.528093695640564, "learning_rate": 8.4e-05, "loss": 0.016095556318759918, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.01623, "step": 22, "tokens/total": 698840, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 0.8761324286460876, "learning_rate": 8.800000000000001e-05, "loss": 0.025180477648973465, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.0255, "step": 23, "tokens/total": 731616, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 1.0102022886276245, "learning_rate": 9.200000000000001e-05, "loss": 0.0362737812101841, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.03694, "step": 24, "tokens/total": 761952, "tokens/train_per_sec_per_gpu": 10.07, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 2.3937389850616455, "learning_rate": 9.6e-05, "loss": 0.04242967069149017, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.04334, "step": 25, "tokens/total": 792240, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 0.9986222982406616, "learning_rate": 0.0001, "loss": 0.026217959821224213, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.02656, "step": 26, "tokens/total": 819696, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 2.0675227642059326, "learning_rate": 9.99990636831587e-05, "loss": 0.03411175683140755, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.0347, "step": 27, "tokens/total": 851176, "tokens/train_per_sec_per_gpu": 10.3, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 0.7148457765579224, "learning_rate": 9.999625477159879e-05, "loss": 0.016825687140226364, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01697, "step": 28, "tokens/total": 886496, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 1.4291799068450928, "learning_rate": 9.999157338221051e-05, "loss": 0.024133075028657913, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.02443, "step": 29, "tokens/total": 918416, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 0.8248594999313354, "learning_rate": 9.998501970980562e-05, "loss": 0.01594935730099678, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.01608, "step": 30, "tokens/total": 948512, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 0.9254230260848999, "learning_rate": 9.997659402710915e-05, "loss": 0.018142005428671837, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.01831, "step": 31, "tokens/total": 982400, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 1.3482874631881714, "learning_rate": 9.996629668474818e-05, "loss": 0.0386526882648468, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.03941, "step": 32, "tokens/total": 1016168, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 0.8922920227050781, "learning_rate": 9.995412811123711e-05, "loss": 0.022911274805665016, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.02318, "step": 33, "tokens/total": 1048376, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 0.6468578577041626, "learning_rate": 9.994008881295999e-05, "loss": 0.02098802663385868, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.02121, "step": 34, "tokens/total": 1080792, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 1.0759810209274292, "learning_rate": 9.992417937414932e-05, "loss": 0.03124830313026905, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.03174, "step": 35, "tokens/total": 1112144, "tokens/train_per_sec_per_gpu": 10.34, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 1.4771379232406616, "learning_rate": 9.99064004568618e-05, "loss": 0.01949450373649597, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.01969, "step": 36, "tokens/total": 1144776, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 0.5574915409088135, "learning_rate": 9.988675280095074e-05, "loss": 0.01168515533208847, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.01175, "step": 37, "tokens/total": 1175576, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.2903294861316681, "learning_rate": 9.986523722403528e-05, "loss": 0.008630835451185703, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00867, "step": 38, "tokens/total": 1206008, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.3810923397541046, "learning_rate": 9.984185462146642e-05, "loss": 0.009537344798445702, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00958, "step": 39, "tokens/total": 1238280, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 0.827174961566925, "learning_rate": 9.98166059662897e-05, "loss": 0.03723431006073952, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.03794, "step": 40, "tokens/total": 1267680, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 1.472495675086975, "learning_rate": 9.978949230920472e-05, "loss": 0.029993321746587753, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.03045, "step": 41, "tokens/total": 1300168, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 1.2033809423446655, "learning_rate": 9.976051477852141e-05, "loss": 0.03761832416057587, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.03833, "step": 42, "tokens/total": 1332960, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 0.5499117374420166, "learning_rate": 9.972967458011312e-05, "loss": 0.030747054144740105, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.03122, "step": 43, "tokens/total": 1365160, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 0.5528761744499207, "learning_rate": 9.96969729973664e-05, "loss": 0.016216395422816277, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.01635, "step": 44, "tokens/total": 1402920, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 20001} +{"epoch": 0.17578125, "grad_norm": 1.134607195854187, "learning_rate": 9.966241139112754e-05, "loss": 0.02275279350578785, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.02301, "step": 45, "tokens/total": 1434976, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 20442} +{"epoch": 0.1796875, "grad_norm": 0.6906723976135254, "learning_rate": 9.96259911996461e-05, "loss": 0.017123041674494743, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01727, "step": 46, "tokens/total": 1466704, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 20874} +{"epoch": 0.18359375, "grad_norm": 0.9817294478416443, "learning_rate": 9.958771393851491e-05, "loss": 0.03285183012485504, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.0334, "step": 47, "tokens/total": 1497856, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 21315} +{"epoch": 0.1875, "grad_norm": 0.6600764989852905, "learning_rate": 9.954758120060702e-05, "loss": 0.03741037845611572, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.03812, "step": 48, "tokens/total": 1529880, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 21775} +{"epoch": 0.19140625, "grad_norm": 0.7678585648536682, "learning_rate": 9.950559465600948e-05, "loss": 0.012658719904720783, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.01274, "step": 49, "tokens/total": 1562128, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 22215} +{"epoch": 0.1953125, "grad_norm": 0.6587523221969604, "learning_rate": 9.946175605195379e-05, "loss": 0.014724966138601303, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01483, "step": 50, "tokens/total": 1597344, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 22700} +{"epoch": 0.19921875, "grad_norm": 0.8285318613052368, "learning_rate": 9.941606721274322e-05, "loss": 0.02108374983072281, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.02131, "step": 51, "tokens/total": 1629896, "tokens/train_per_sec_per_gpu": 6.95, "tokens/trainable": 23135} +{"epoch": 0.203125, "grad_norm": 0.6512331366539001, "learning_rate": 9.936853003967685e-05, "loss": 0.01495690830051899, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.01507, "step": 52, "tokens/total": 1663816, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 2.0477070808410645, "learning_rate": 9.93191465109705e-05, "loss": 0.03486660122871399, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.03548, "step": 53, "tokens/total": 1696744, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 1.3231333494186401, "learning_rate": 9.926791868167438e-05, "loss": 0.02064744383096695, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.02086, "step": 54, "tokens/total": 1731552, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 0.8804490566253662, "learning_rate": 9.921484868358753e-05, "loss": 0.04345439746975899, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.04441, "step": 55, "tokens/total": 1763208, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 0.9544849395751953, "learning_rate": 9.915993872516924e-05, "loss": 0.019646326079964638, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.01984, "step": 56, "tokens/total": 1796120, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.7176945805549622, "learning_rate": 9.9103191091447e-05, "loss": 0.005182445049285889, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.0052, "step": 57, "tokens/total": 1826048, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 4.232582092285156, "learning_rate": 9.904460814392147e-05, "loss": 0.009594711475074291, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.00964, "step": 58, "tokens/total": 1863152, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.6546913385391235, "learning_rate": 9.898419232046825e-05, "loss": 0.015548648312687874, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01567, "step": 59, "tokens/total": 1898008, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.4972032308578491, "learning_rate": 9.892194613523633e-05, "loss": 0.005309485364705324, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00532, "step": 60, "tokens/total": 1932616, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.509767472743988, "learning_rate": 9.885787217854357e-05, "loss": 0.008575799874961376, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00861, "step": 61, "tokens/total": 1964544, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.28598740696907043, "learning_rate": 9.879197311676887e-05, "loss": 0.012800229713320732, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.01288, "step": 62, "tokens/total": 1997272, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.9693397283554077, "learning_rate": 9.872425169224113e-05, "loss": 0.024132709950208664, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.02443, "step": 63, "tokens/total": 2029744, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 28623} +{"epoch": 0.25, "grad_norm": 0.6161311864852905, "learning_rate": 9.865471072312528e-05, "loss": 0.01906217634677887, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.01925, "step": 64, "tokens/total": 2065136, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 29108} +{"epoch": 0.25390625, "grad_norm": 0.49687281250953674, "learning_rate": 9.858335310330492e-05, "loss": 0.007149277720600367, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00717, "step": 65, "tokens/total": 2094864, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 29570} +{"epoch": 0.2578125, "grad_norm": 0.6245746612548828, "learning_rate": 9.851018180226185e-05, "loss": 0.009866941720247269, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00992, "step": 66, "tokens/total": 2126864, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 30011} +{"epoch": 0.26171875, "grad_norm": 0.32380324602127075, "learning_rate": 9.843519986495259e-05, "loss": 0.005770830903202295, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00579, "step": 67, "tokens/total": 2157816, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 30446} +{"epoch": 0.265625, "grad_norm": 0.4680977761745453, "learning_rate": 9.835841041168162e-05, "loss": 0.008309643715620041, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00834, "step": 68, "tokens/total": 2192664, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 30913} +{"epoch": 0.26953125, "grad_norm": 0.31535544991493225, "learning_rate": 9.82798166379715e-05, "loss": 0.00846133567392826, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.0085, "step": 69, "tokens/total": 2223792, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.45732975006103516, "learning_rate": 9.819942181443002e-05, "loss": 0.012252659536898136, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.01233, "step": 70, "tokens/total": 2256288, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 31844} +{"epoch": 0.27734375, "grad_norm": 0.2576862871646881, "learning_rate": 9.811722928661392e-05, "loss": 0.004620684310793877, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.00463, "step": 71, "tokens/total": 2284168, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 2.3975791931152344, "learning_rate": 9.803324247488975e-05, "loss": 0.008411853574216366, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00845, "step": 72, "tokens/total": 2316496, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.1374823898077011, "learning_rate": 9.794746487429161e-05, "loss": 0.0007019452750682831, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.0007, "step": 73, "tokens/total": 2347680, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 1.7289013862609863, "learning_rate": 9.785990005437554e-05, "loss": 0.02826273813843727, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.02867, "step": 74, "tokens/total": 2380912, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.660820722579956, "learning_rate": 9.777055165907117e-05, "loss": 0.013953514397144318, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.01405, "step": 75, "tokens/total": 2412000, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.04904603585600853, "learning_rate": 9.767942340652993e-05, "loss": 0.0005219369195401669, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00052, "step": 76, "tokens/total": 2441776, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.727301299571991, "learning_rate": 9.758651908897035e-05, "loss": 0.003764320630580187, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00377, "step": 77, "tokens/total": 2473896, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 34894} +{"epoch": 0.3046875, "grad_norm": 2.168595790863037, "learning_rate": 9.749184257252033e-05, "loss": 0.04962445795536041, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.05088, "step": 78, "tokens/total": 2505568, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 35334} +{"epoch": 0.30859375, "grad_norm": 1.4368618726730347, "learning_rate": 9.739539779705614e-05, "loss": 0.03494219481945038, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.03556, "step": 79, "tokens/total": 2542120, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 35829} +{"epoch": 0.3125, "grad_norm": 0.793967604637146, "learning_rate": 9.729718877603861e-05, "loss": 0.016915686428546906, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.01706, "step": 80, "tokens/total": 2572712, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 36260} +{"epoch": 0.31640625, "grad_norm": 1.0048654079437256, "learning_rate": 9.719721959634592e-05, "loss": 0.024288304150104523, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.02459, "step": 81, "tokens/total": 2604304, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 36697} +{"epoch": 0.3203125, "grad_norm": 0.4293708801269531, "learning_rate": 9.709549441810375e-05, "loss": 0.013622366823256016, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.01372, "step": 82, "tokens/total": 2639072, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 37181} +{"epoch": 0.32421875, "grad_norm": 1.0458003282546997, "learning_rate": 9.699201747451195e-05, "loss": 0.033958952873945236, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.03454, "step": 83, "tokens/total": 2670240, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 37629} +{"epoch": 0.328125, "grad_norm": 0.44647032022476196, "learning_rate": 9.688679307166854e-05, "loss": 0.008412299677729607, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00845, "step": 84, "tokens/total": 2701112, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 38092} +{"epoch": 0.33203125, "grad_norm": 0.40073704719543457, "learning_rate": 9.677982558839042e-05, "loss": 0.02143809385597706, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.02167, "step": 85, "tokens/total": 2733488, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 38571} +{"epoch": 0.3359375, "grad_norm": 0.2773541510105133, "learning_rate": 9.66711194760312e-05, "loss": 0.012801357544958591, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01288, "step": 86, "tokens/total": 2769880, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 39059} +{"epoch": 0.33984375, "grad_norm": 6.509273052215576, "learning_rate": 9.656067925829593e-05, "loss": 0.02613883838057518, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.02648, "step": 87, "tokens/total": 2799384, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 39519} +{"epoch": 0.34375, "grad_norm": 0.25664272904396057, "learning_rate": 9.644850953105288e-05, "loss": 0.010865864343941212, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.01093, "step": 88, "tokens/total": 2828608, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 39961} +{"epoch": 0.34765625, "grad_norm": 0.4949970841407776, "learning_rate": 9.633461496214225e-05, "loss": 0.014130642637610435, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.01423, "step": 89, "tokens/total": 2860464, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 40431} +{"epoch": 0.3515625, "grad_norm": 1.4668720960617065, "learning_rate": 9.621900029118195e-05, "loss": 0.03554742783308029, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.03619, "step": 90, "tokens/total": 2889528, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 40854} +{"epoch": 0.35546875, "grad_norm": 0.3460696041584015, "learning_rate": 9.610167032937036e-05, "loss": 0.005880081094801426, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.0059, "step": 91, "tokens/total": 2923880, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 41373} +{"epoch": 0.359375, "grad_norm": 0.3303653597831726, "learning_rate": 9.598262995928611e-05, "loss": 0.0031036220025271177, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00311, "step": 92, "tokens/total": 2953944, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 41842} +{"epoch": 0.36328125, "grad_norm": 0.4414062798023224, "learning_rate": 9.586188413468492e-05, "loss": 0.006893655750900507, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00692, "step": 93, "tokens/total": 2988120, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 42288} +{"epoch": 0.3671875, "grad_norm": 0.4796414375305176, "learning_rate": 9.57394378802934e-05, "loss": 0.0096715884283185, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00972, "step": 94, "tokens/total": 3020416, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 42740} +{"epoch": 0.37109375, "grad_norm": 0.9419648051261902, "learning_rate": 9.56152962916e-05, "loss": 0.012559151276946068, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.01264, "step": 95, "tokens/total": 3048800, "tokens/train_per_sec_per_gpu": 10.06, "tokens/trainable": 43138} +{"epoch": 0.375, "grad_norm": 1.0440526008605957, "learning_rate": 9.548946453464296e-05, "loss": 0.006730351597070694, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00675, "step": 96, "tokens/total": 3081768, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 43623} +{"epoch": 0.37890625, "grad_norm": 6.57992696762085, "learning_rate": 9.53619478457953e-05, "loss": 0.02622966840863228, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.02658, "step": 97, "tokens/total": 3115608, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 44079} +{"epoch": 0.3828125, "grad_norm": 0.6222487092018127, "learning_rate": 9.523275153154695e-05, "loss": 0.008357911370694637, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00839, "step": 98, "tokens/total": 3146880, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 44529} +{"epoch": 0.38671875, "grad_norm": 3.804287910461426, "learning_rate": 9.51018809682839e-05, "loss": 0.02821996435523033, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.02862, "step": 99, "tokens/total": 3180088, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 44976} +{"epoch": 0.390625, "grad_norm": 0.7035965919494629, "learning_rate": 9.49693416020645e-05, "loss": 0.020511023700237274, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.02072, "step": 100, "tokens/total": 3213968, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 45429} +{"epoch": 0.39453125, "grad_norm": 0.32482975721359253, "learning_rate": 9.483513894839276e-05, "loss": 0.004517541266977787, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00453, "step": 101, "tokens/total": 3245976, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 45876} +{"epoch": 0.3984375, "grad_norm": 1.301345944404602, "learning_rate": 9.469927859198888e-05, "loss": 0.0184236541390419, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01859, "step": 102, "tokens/total": 3278792, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 46363} +{"epoch": 0.40234375, "grad_norm": 0.6991849541664124, "learning_rate": 9.456176618655689e-05, "loss": 0.027093172073364258, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.02746, "step": 103, "tokens/total": 3309360, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 46810} +{"epoch": 0.40625, "grad_norm": 0.3233588635921478, "learning_rate": 9.442260745454927e-05, "loss": 0.00600785668939352, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00603, "step": 104, "tokens/total": 3344344, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 47287} +{"epoch": 0.41015625, "grad_norm": 0.5362960696220398, "learning_rate": 9.428180818692884e-05, "loss": 0.010858455672860146, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.01092, "step": 105, "tokens/total": 3378832, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 47788} +{"epoch": 0.4140625, "grad_norm": 0.663091778755188, "learning_rate": 9.413937424292791e-05, "loss": 0.006703128106892109, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00673, "step": 106, "tokens/total": 3415416, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 48243} +{"epoch": 0.41796875, "grad_norm": 0.29907384514808655, "learning_rate": 9.399531154980424e-05, "loss": 0.008487301878631115, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00852, "step": 107, "tokens/total": 3448120, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 48692} +{"epoch": 0.421875, "grad_norm": 0.31643715500831604, "learning_rate": 9.384962610259455e-05, "loss": 0.003412924474105239, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00342, "step": 108, "tokens/total": 3480800, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.2695532441139221, "learning_rate": 9.370232396386494e-05, "loss": 0.004823336377739906, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00483, "step": 109, "tokens/total": 3509992, "tokens/train_per_sec_per_gpu": 10.4, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 1.0287384986877441, "learning_rate": 9.355341126345868e-05, "loss": 0.017217569053173065, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01737, "step": 110, "tokens/total": 3542872, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.03502413630485535, "learning_rate": 9.340289419824107e-05, "loss": 0.00034347377368249, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.00034, "step": 111, "tokens/total": 3573896, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.8728456497192383, "learning_rate": 9.325077903184159e-05, "loss": 0.003663273062556982, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00367, "step": 112, "tokens/total": 3605648, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.07054361701011658, "learning_rate": 9.30970720943932e-05, "loss": 0.0004851872508879751, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00049, "step": 113, "tokens/total": 3638544, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 51389} +{"epoch": 0.4453125, "grad_norm": 0.025585580617189407, "learning_rate": 9.2941779782269e-05, "loss": 0.000143700948683545, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00014, "step": 114, "tokens/total": 3670776, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 51862} +{"epoch": 0.44921875, "grad_norm": 0.5112435221672058, "learning_rate": 9.278490855781596e-05, "loss": 0.004419410601258278, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.00443, "step": 115, "tokens/total": 3701472, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 52301} +{"epoch": 0.453125, "grad_norm": 2.69728684425354, "learning_rate": 9.262646494908604e-05, "loss": 0.01188542041927576, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.01196, "step": 116, "tokens/total": 3735872, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 52755} +{"epoch": 0.45703125, "grad_norm": 0.8919106125831604, "learning_rate": 9.246645554956457e-05, "loss": 0.03633888065814972, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.03701, "step": 117, "tokens/total": 3770032, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 53224} +{"epoch": 0.4609375, "grad_norm": 2.279548168182373, "learning_rate": 9.230488701789578e-05, "loss": 0.022335728630423546, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.02259, "step": 118, "tokens/total": 3805264, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 53666} +{"epoch": 0.46484375, "grad_norm": 0.286337286233902, "learning_rate": 9.214176607760577e-05, "loss": 0.001185867004096508, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00119, "step": 119, "tokens/total": 3840776, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 54129} +{"epoch": 0.46875, "grad_norm": 0.7278910279273987, "learning_rate": 9.197709951682268e-05, "loss": 0.01593283750116825, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.01606, "step": 120, "tokens/total": 3871376, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 54587} +{"epoch": 0.47265625, "grad_norm": 1.343673586845398, "learning_rate": 9.181089418799428e-05, "loss": 0.023560095578432083, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.02384, "step": 121, "tokens/total": 3904552, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 55070} +{"epoch": 0.4765625, "grad_norm": 0.3144819140434265, "learning_rate": 9.164315700760271e-05, "loss": 0.00787513330578804, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00791, "step": 122, "tokens/total": 3937344, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 55494} +{"epoch": 0.48046875, "grad_norm": 0.5430300831794739, "learning_rate": 9.147389495587671e-05, "loss": 0.0178750641644001, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01804, "step": 123, "tokens/total": 3967392, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 55940} +{"epoch": 0.484375, "grad_norm": 0.35953861474990845, "learning_rate": 9.130311507650116e-05, "loss": 0.005965707823634148, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00598, "step": 124, "tokens/total": 3998280, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 56404} +{"epoch": 0.48828125, "grad_norm": 0.2871270477771759, "learning_rate": 9.113082447632394e-05, "loss": 0.012364400550723076, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.01244, "step": 125, "tokens/total": 4029616, "tokens/train_per_sec_per_gpu": 10.55, "tokens/trainable": 56878} +{"epoch": 0.4921875, "grad_norm": 0.40631943941116333, "learning_rate": 9.09570303250602e-05, "loss": 0.009885433129966259, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00993, "step": 126, "tokens/total": 4060368, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 57362} +{"epoch": 0.49609375, "grad_norm": 0.39274629950523376, "learning_rate": 9.078173985499394e-05, "loss": 0.006389113143086433, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00641, "step": 127, "tokens/total": 4089688, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 57792} +{"epoch": 0.5, "grad_norm": 0.5778021216392517, "learning_rate": 9.060496036067713e-05, "loss": 0.011874567717313766, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.01195, "step": 128, "tokens/total": 4125048, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 58216} +{"epoch": 0.50390625, "grad_norm": 0.4167674481868744, "learning_rate": 9.042669919862615e-05, "loss": 0.008639799430966377, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00868, "step": 129, "tokens/total": 4156488, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 58680} +{"epoch": 0.5078125, "grad_norm": 0.47027847170829773, "learning_rate": 9.024696378701557e-05, "loss": 0.01061214692890644, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.01067, "step": 130, "tokens/total": 4189344, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 59113} +{"epoch": 0.51171875, "grad_norm": 0.9361854791641235, "learning_rate": 9.006576160536948e-05, "loss": 0.009113864041864872, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00916, "step": 131, "tokens/total": 4220768, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 59562} +{"epoch": 0.515625, "grad_norm": 0.8266016840934753, "learning_rate": 8.988310019425035e-05, "loss": 0.017906004562973976, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.01807, "step": 132, "tokens/total": 4256880, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 60006} +{"epoch": 0.51953125, "grad_norm": 0.11345213651657104, "learning_rate": 8.969898715494506e-05, "loss": 0.00174340745434165, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00174, "step": 133, "tokens/total": 4290928, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 60487} +{"epoch": 0.5234375, "grad_norm": 0.6843398213386536, "learning_rate": 8.951343014914869e-05, "loss": 0.012469355016946793, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.01255, "step": 134, "tokens/total": 4325728, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 60953} +{"epoch": 0.52734375, "grad_norm": 0.9663278460502625, "learning_rate": 8.932643689864568e-05, "loss": 0.010913371108472347, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.01097, "step": 135, "tokens/total": 4358848, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 61440} +{"epoch": 0.53125, "grad_norm": 0.47949734330177307, "learning_rate": 8.913801518498845e-05, "loss": 0.0053008622489869595, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00531, "step": 136, "tokens/total": 4393504, "tokens/train_per_sec_per_gpu": 6.93, "tokens/trainable": 61901} +{"epoch": 0.53515625, "grad_norm": 4.154111385345459, "learning_rate": 8.894817284917364e-05, "loss": 0.010648422874510288, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.01071, "step": 137, "tokens/total": 4427912, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 62355} +{"epoch": 0.5390625, "grad_norm": 0.771457850933075, "learning_rate": 8.875691779131569e-05, "loss": 0.013175553642213345, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01326, "step": 138, "tokens/total": 4462448, "tokens/train_per_sec_per_gpu": 6.44, "tokens/trainable": 62786} +{"epoch": 0.54296875, "grad_norm": 0.24782498180866241, "learning_rate": 8.856425797031829e-05, "loss": 0.0018043633317574859, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00181, "step": 139, "tokens/total": 4495992, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 63228} +{"epoch": 0.546875, "grad_norm": 1.5244020223617554, "learning_rate": 8.837020140354295e-05, "loss": 0.010667708702385426, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.01072, "step": 140, "tokens/total": 4529600, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 63693} +{"epoch": 0.55078125, "grad_norm": 0.5065008401870728, "learning_rate": 8.817475616647554e-05, "loss": 0.01055715512484312, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.01061, "step": 141, "tokens/total": 4562536, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 64159} +{"epoch": 0.5546875, "grad_norm": 0.4658786654472351, "learning_rate": 8.797793039239017e-05, "loss": 0.009148654527962208, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00919, "step": 142, "tokens/total": 4595232, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 64608} +{"epoch": 0.55859375, "grad_norm": 2.4970381259918213, "learning_rate": 8.777973227201069e-05, "loss": 0.01991599053144455, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.02012, "step": 143, "tokens/total": 4629712, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 65117} +{"epoch": 0.5625, "grad_norm": 0.3903602361679077, "learning_rate": 8.758017005316988e-05, "loss": 0.013442914932966232, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.01353, "step": 144, "tokens/total": 4661752, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 65542} +{"epoch": 0.56640625, "grad_norm": 0.26226574182510376, "learning_rate": 8.737925204046629e-05, "loss": 0.010091948322951794, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.01014, "step": 145, "tokens/total": 4694416, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 65992} +{"epoch": 0.5703125, "grad_norm": 0.34129631519317627, "learning_rate": 8.717698659491851e-05, "loss": 0.00600326806306839, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00602, "step": 146, "tokens/total": 4728264, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 66449} +{"epoch": 0.57421875, "grad_norm": 0.36804917454719543, "learning_rate": 8.697338213361735e-05, "loss": 0.008204538375139236, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00824, "step": 147, "tokens/total": 4758952, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 66897} +{"epoch": 0.578125, "grad_norm": 0.49535271525382996, "learning_rate": 8.676844712937552e-05, "loss": 0.010706916451454163, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.01076, "step": 148, "tokens/total": 4795624, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 67369} +{"epoch": 0.58203125, "grad_norm": 0.5420557260513306, "learning_rate": 8.656219011037509e-05, "loss": 0.013655577786266804, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01375, "step": 149, "tokens/total": 4828976, "tokens/train_per_sec_per_gpu": 6.65, "tokens/trainable": 67857} +{"epoch": 0.5859375, "grad_norm": 0.12776009738445282, "learning_rate": 8.63546196598125e-05, "loss": 0.004056468140333891, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00406, "step": 150, "tokens/total": 4862128, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 68305} +{"epoch": 0.58984375, "grad_norm": 0.21698588132858276, "learning_rate": 8.614574441554145e-05, "loss": 0.005862970370799303, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00588, "step": 151, "tokens/total": 4892760, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 68730} +{"epoch": 0.59375, "grad_norm": 1.148898720741272, "learning_rate": 8.593557306971349e-05, "loss": 0.015088299289345741, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.0152, "step": 152, "tokens/total": 4925296, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 69185} +{"epoch": 0.59765625, "grad_norm": 0.2330581247806549, "learning_rate": 8.572411436841618e-05, "loss": 0.007038078736513853, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00706, "step": 153, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 69660} +{"epoch": 0.6015625, "grad_norm": 0.3299039304256439, "learning_rate": 8.551137711130922e-05, "loss": 0.006851613521575928, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00688, "step": 154, "tokens/total": 4985520, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 70066} +{"epoch": 0.60546875, "grad_norm": 0.1594575047492981, "learning_rate": 8.529737015125824e-05, "loss": 0.003128394018858671, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00313, "step": 155, "tokens/total": 5019320, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 70532} +{"epoch": 0.609375, "grad_norm": 1.090949535369873, "learning_rate": 8.508210239396639e-05, "loss": 0.029943590983748436, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.0304, "step": 156, "tokens/total": 5052056, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 70981} +{"epoch": 0.61328125, "grad_norm": 0.5347111821174622, "learning_rate": 8.486558279760375e-05, "loss": 0.002775952685624361, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00278, "step": 157, "tokens/total": 5082280, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 71407} +{"epoch": 0.6171875, "grad_norm": 0.35970360040664673, "learning_rate": 8.464782037243449e-05, "loss": 0.012963389977812767, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01305, "step": 158, "tokens/total": 5113912, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 71848} +{"epoch": 0.62109375, "grad_norm": 0.09817509353160858, "learning_rate": 8.442882418044202e-05, "loss": 0.0026100678369402885, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00261, "step": 159, "tokens/total": 5145200, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 72325} +{"epoch": 0.625, "grad_norm": 0.3817746937274933, "learning_rate": 8.420860333495179e-05, "loss": 0.00787435844540596, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00791, "step": 160, "tokens/total": 5174928, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 72739} +{"epoch": 0.62890625, "grad_norm": 0.12796883285045624, "learning_rate": 8.398716700025208e-05, "loss": 0.001635900465771556, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00164, "step": 161, "tokens/total": 5206888, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 73191} +{"epoch": 0.6328125, "grad_norm": 0.6142289042472839, "learning_rate": 8.376452439121266e-05, "loss": 0.012117969803512096, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.01219, "step": 162, "tokens/total": 5236336, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 73642} +{"epoch": 0.63671875, "grad_norm": 0.3273414373397827, "learning_rate": 8.354068477290124e-05, "loss": 0.006811546161770821, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00683, "step": 163, "tokens/total": 5267760, "tokens/train_per_sec_per_gpu": 10.2, "tokens/trainable": 74101} +{"epoch": 0.640625, "grad_norm": 0.61309415102005, "learning_rate": 8.331565746019807e-05, "loss": 0.007880308665335178, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00791, "step": 164, "tokens/total": 5298896, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 74558} +{"epoch": 0.64453125, "grad_norm": 0.4478000998497009, "learning_rate": 8.308945181740812e-05, "loss": 0.010343093425035477, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.0104, "step": 165, "tokens/total": 5330792, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 75000} +{"epoch": 0.6484375, "grad_norm": 0.4768712818622589, "learning_rate": 8.286207725787153e-05, "loss": 0.005799765698611736, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00582, "step": 166, "tokens/total": 5364144, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 75445} +{"epoch": 0.65234375, "grad_norm": 0.5430775284767151, "learning_rate": 8.263354324357182e-05, "loss": 0.014093572273850441, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.01419, "step": 167, "tokens/total": 5396656, "tokens/train_per_sec_per_gpu": 6.87, "tokens/trainable": 75880} +{"epoch": 0.65625, "grad_norm": 0.4106767475605011, "learning_rate": 8.240385928474219e-05, "loss": 0.008665701374411583, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.0087, "step": 168, "tokens/total": 5427736, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 76294} +{"epoch": 0.66015625, "grad_norm": 0.3590318560600281, "learning_rate": 8.217303493946967e-05, "loss": 0.00690571591258049, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00693, "step": 169, "tokens/total": 5461752, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 76742} +{"epoch": 0.6640625, "grad_norm": 0.5233500599861145, "learning_rate": 8.194107981329746e-05, "loss": 0.006049932911992073, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00607, "step": 170, "tokens/total": 5498688, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 77196} +{"epoch": 0.66796875, "grad_norm": 0.038191165775060654, "learning_rate": 8.170800355882518e-05, "loss": 0.0004784919728990644, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00048, "step": 171, "tokens/total": 5530936, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 77663} +{"epoch": 0.671875, "grad_norm": 0.3231879472732544, "learning_rate": 8.147381587530713e-05, "loss": 0.0042290943674743176, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00424, "step": 172, "tokens/total": 5564176, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 78117} +{"epoch": 0.67578125, "grad_norm": 0.1314975917339325, "learning_rate": 8.123852650824877e-05, "loss": 0.0018854814115911722, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00189, "step": 173, "tokens/total": 5596856, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 78601} +{"epoch": 0.6796875, "grad_norm": 0.7116119861602783, "learning_rate": 8.100214524900103e-05, "loss": 0.005934853106737137, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00595, "step": 174, "tokens/total": 5630016, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 79032} +{"epoch": 0.68359375, "grad_norm": 0.14405472576618195, "learning_rate": 8.076468193435301e-05, "loss": 0.0017410372383892536, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00174, "step": 175, "tokens/total": 5659656, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 79456} +{"epoch": 0.6875, "grad_norm": 0.48993155360221863, "learning_rate": 8.052614644612253e-05, "loss": 0.00463218055665493, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00464, "step": 176, "tokens/total": 5692408, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 79917} +{"epoch": 0.69140625, "grad_norm": 0.20136138796806335, "learning_rate": 8.028654871074489e-05, "loss": 0.001354968291707337, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00136, "step": 177, "tokens/total": 5724672, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 80358} +{"epoch": 0.6953125, "grad_norm": 0.02019348554313183, "learning_rate": 8.004589869885986e-05, "loss": 0.00018824526341632009, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00019, "step": 178, "tokens/total": 5754384, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 80777} +{"epoch": 0.69921875, "grad_norm": 3.238379716873169, "learning_rate": 7.980420642489674e-05, "loss": 0.014210407622158527, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01431, "step": 179, "tokens/total": 5789344, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 81250} +{"epoch": 0.703125, "grad_norm": 0.0772034078836441, "learning_rate": 7.95614819466576e-05, "loss": 0.0005619146977551281, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00056, "step": 180, "tokens/total": 5821816, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 81715} +{"epoch": 0.70703125, "grad_norm": 0.3522290289402008, "learning_rate": 7.931773536489872e-05, "loss": 0.0019669991452246904, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00197, "step": 181, "tokens/total": 5854168, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 82160} +{"epoch": 0.7109375, "grad_norm": 2.7761714458465576, "learning_rate": 7.907297682291035e-05, "loss": 0.045900747179985046, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.04697, "step": 182, "tokens/total": 5886064, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 82624} +{"epoch": 0.71484375, "grad_norm": 0.49149289727211, "learning_rate": 7.882721650609442e-05, "loss": 0.0073412638157606125, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00737, "step": 183, "tokens/total": 5921648, "tokens/train_per_sec_per_gpu": 6.74, "tokens/trainable": 83125} +{"epoch": 0.71875, "grad_norm": 0.7061439752578735, "learning_rate": 7.85804646415409e-05, "loss": 0.006433587986975908, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.00645, "step": 184, "tokens/total": 5954952, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 83581} +{"epoch": 0.72265625, "grad_norm": 0.13410162925720215, "learning_rate": 7.833273149760207e-05, "loss": 0.0010352074168622494, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00104, "step": 185, "tokens/total": 5985432, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 84051} +{"epoch": 0.7265625, "grad_norm": 1.5294954776763916, "learning_rate": 7.808402738346527e-05, "loss": 0.01805983856320381, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01822, "step": 186, "tokens/total": 6020664, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 84500} +{"epoch": 0.73046875, "grad_norm": 0.8359888792037964, "learning_rate": 7.783436264872382e-05, "loss": 0.0014164319727569818, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00142, "step": 187, "tokens/total": 6055024, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 84986} +{"epoch": 0.734375, "grad_norm": 0.9199267625808716, "learning_rate": 7.758374768294647e-05, "loss": 0.021248940378427505, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.02148, "step": 188, "tokens/total": 6088296, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 85465} +{"epoch": 0.73828125, "grad_norm": 0.44393062591552734, "learning_rate": 7.733219291524489e-05, "loss": 0.007755171973258257, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00779, "step": 189, "tokens/total": 6119840, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 85896} +{"epoch": 0.7421875, "grad_norm": 0.36129093170166016, "learning_rate": 7.707970881383977e-05, "loss": 0.003800515318289399, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.00381, "step": 190, "tokens/total": 6149880, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 86346} +{"epoch": 0.74609375, "grad_norm": 0.8958450555801392, "learning_rate": 7.682630588562518e-05, "loss": 0.006366039626300335, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00639, "step": 191, "tokens/total": 6182904, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 86777} +{"epoch": 0.75, "grad_norm": 0.26846015453338623, "learning_rate": 7.657199467573129e-05, "loss": 0.0029315222054719925, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00294, "step": 192, "tokens/total": 6214144, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 87249} +{"epoch": 0.75390625, "grad_norm": 0.6828296780586243, "learning_rate": 7.631678576708561e-05, "loss": 0.01605869084596634, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.01619, "step": 193, "tokens/total": 6247072, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 87741} +{"epoch": 0.7578125, "grad_norm": 0.2400084137916565, "learning_rate": 7.606068977997255e-05, "loss": 0.0034776744432747364, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00348, "step": 194, "tokens/total": 6281112, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 88219} +{"epoch": 0.76171875, "grad_norm": 0.3966066241264343, "learning_rate": 7.580371737159148e-05, "loss": 0.011279745027422905, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.01134, "step": 195, "tokens/total": 6310200, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 88630} +{"epoch": 0.765625, "grad_norm": 0.21670988202095032, "learning_rate": 7.554587923561324e-05, "loss": 0.0016269105253741145, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00163, "step": 196, "tokens/total": 6341792, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 89068} +{"epoch": 0.76953125, "grad_norm": 0.63811856508255, "learning_rate": 7.528718610173511e-05, "loss": 0.01718849129974842, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.01734, "step": 197, "tokens/total": 6374040, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 89489} +{"epoch": 0.7734375, "grad_norm": 0.05122731253504753, "learning_rate": 7.502764873523431e-05, "loss": 0.0006090165697969496, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00061, "step": 198, "tokens/total": 6405552, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 89928} +{"epoch": 0.77734375, "grad_norm": 0.15850932896137238, "learning_rate": 7.476727793652011e-05, "loss": 0.005108004901558161, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00512, "step": 199, "tokens/total": 6436016, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 90407} +{"epoch": 0.78125, "grad_norm": 0.21040640771389008, "learning_rate": 7.450608454068415e-05, "loss": 0.0066337380558252335, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00666, "step": 200, "tokens/total": 6468656, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 90849} +{"epoch": 0.78515625, "grad_norm": 0.12679724395275116, "learning_rate": 7.424407941704987e-05, "loss": 0.0037224176339805126, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00373, "step": 201, "tokens/total": 6502728, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 91350} +{"epoch": 0.7890625, "grad_norm": 0.4564002752304077, "learning_rate": 7.398127346871986e-05, "loss": 0.012911060824990273, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.01299, "step": 202, "tokens/total": 6535616, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 91805} +{"epoch": 0.79296875, "grad_norm": 0.406088650226593, "learning_rate": 7.371767763212238e-05, "loss": 0.0084202466532588, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00846, "step": 203, "tokens/total": 6567088, "tokens/train_per_sec_per_gpu": 10.09, "tokens/trainable": 92258} +{"epoch": 0.796875, "grad_norm": 0.09627223759889603, "learning_rate": 7.345330287655617e-05, "loss": 0.0028775562532246113, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00288, "step": 204, "tokens/total": 6600808, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 92721} +{"epoch": 0.80078125, "grad_norm": 0.19935952126979828, "learning_rate": 7.31881602037339e-05, "loss": 0.004568814300000668, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.00458, "step": 205, "tokens/total": 6631384, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 93160} +{"epoch": 0.8046875, "grad_norm": 1.076123595237732, "learning_rate": 7.29222606473245e-05, "loss": 0.011253963224589825, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.01132, "step": 206, "tokens/total": 6661888, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 93605} +{"epoch": 0.80859375, "grad_norm": 0.7598662376403809, "learning_rate": 7.265561527249383e-05, "loss": 0.011119290255010128, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.01118, "step": 207, "tokens/total": 6693456, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 94041} +{"epoch": 0.8125, "grad_norm": 0.29415762424468994, "learning_rate": 7.238823517544436e-05, "loss": 0.010499201714992523, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01055, "step": 208, "tokens/total": 6727840, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 94512} +{"epoch": 0.81640625, "grad_norm": 0.0325012281537056, "learning_rate": 7.212013148295333e-05, "loss": 0.0005531870992854238, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00055, "step": 209, "tokens/total": 6761704, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 94994} +{"epoch": 0.8203125, "grad_norm": 0.17431612312793732, "learning_rate": 7.185131535190975e-05, "loss": 0.004647121299058199, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00466, "step": 210, "tokens/total": 6792648, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 95503} +{"epoch": 0.82421875, "grad_norm": 0.09196870028972626, "learning_rate": 7.158179796885005e-05, "loss": 0.001705814735032618, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00171, "step": 211, "tokens/total": 6824424, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 95954} +{"epoch": 0.828125, "grad_norm": 0.33152514696121216, "learning_rate": 7.131159054949273e-05, "loss": 0.00476846843957901, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00478, "step": 212, "tokens/total": 6861216, "tokens/train_per_sec_per_gpu": 7.12, "tokens/trainable": 96428} +{"epoch": 0.83203125, "grad_norm": 0.3385855555534363, "learning_rate": 7.104070433827139e-05, "loss": 0.006705946289002895, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00673, "step": 213, "tokens/total": 6892704, "tokens/train_per_sec_per_gpu": 7.24, "tokens/trainable": 96828} +{"epoch": 0.8359375, "grad_norm": 0.14400030672550201, "learning_rate": 7.076915060786705e-05, "loss": 0.0015542471082881093, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.00156, "step": 214, "tokens/total": 6924816, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 97270} +{"epoch": 0.83984375, "grad_norm": 0.05728885531425476, "learning_rate": 7.049694065873882e-05, "loss": 0.0007041757926344872, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.0007, "step": 215, "tokens/total": 6957536, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 97727} +{"epoch": 0.84375, "grad_norm": 0.35367095470428467, "learning_rate": 7.022408581865382e-05, "loss": 0.0067504337057471275, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00677, "step": 216, "tokens/total": 6991448, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 98201} +{"epoch": 0.84765625, "grad_norm": 0.8292517066001892, "learning_rate": 6.99505974422157e-05, "loss": 0.0014603568706661463, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00146, "step": 217, "tokens/total": 7023712, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 98674} +{"epoch": 0.8515625, "grad_norm": 0.11045040190219879, "learning_rate": 6.967648691039213e-05, "loss": 0.001053584273904562, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00105, "step": 218, "tokens/total": 7055040, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 99108} +{"epoch": 0.85546875, "grad_norm": 0.16487593948841095, "learning_rate": 6.940176563004123e-05, "loss": 0.002009279327467084, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00201, "step": 219, "tokens/total": 7084944, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 99540} +{"epoch": 0.859375, "grad_norm": 0.10797303169965744, "learning_rate": 6.912644503343682e-05, "loss": 0.0012501373421400785, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00125, "step": 220, "tokens/total": 7119264, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 99978} +{"epoch": 0.86328125, "grad_norm": 0.321428507566452, "learning_rate": 6.885053657779273e-05, "loss": 0.007450643461197615, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.00748, "step": 221, "tokens/total": 7150848, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 100450} +{"epoch": 0.8671875, "grad_norm": 0.14097267389297485, "learning_rate": 6.857405174478604e-05, "loss": 0.000721803167834878, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00072, "step": 222, "tokens/total": 7185176, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 100903} +{"epoch": 0.87109375, "grad_norm": 0.9277680516242981, "learning_rate": 6.82970020400792e-05, "loss": 0.02284649945795536, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.02311, "step": 223, "tokens/total": 7219024, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 101351} +{"epoch": 0.875, "grad_norm": 0.26121583580970764, "learning_rate": 6.801939899284132e-05, "loss": 0.004581470042467117, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00459, "step": 224, "tokens/total": 7253816, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 101857} +{"epoch": 0.87890625, "grad_norm": 0.33894532918930054, "learning_rate": 6.774125415526827e-05, "loss": 0.006552088540047407, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00657, "step": 225, "tokens/total": 7286344, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 102328} +{"epoch": 0.8828125, "grad_norm": 0.026581229642033577, "learning_rate": 6.746257910210214e-05, "loss": 0.000254235346801579, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00025, "step": 226, "tokens/total": 7317952, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 102779} +{"epoch": 0.88671875, "grad_norm": 0.09273765981197357, "learning_rate": 6.718338543014937e-05, "loss": 0.0008810809813439846, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00088, "step": 227, "tokens/total": 7347336, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 103228} +{"epoch": 0.890625, "grad_norm": 0.17798450589179993, "learning_rate": 6.69036847577983e-05, "loss": 0.0028134118765592575, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00282, "step": 228, "tokens/total": 7377592, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 103684} +{"epoch": 0.89453125, "grad_norm": 0.33949822187423706, "learning_rate": 6.662348872453553e-05, "loss": 0.010466844774782658, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.01052, "step": 229, "tokens/total": 7410184, "tokens/train_per_sec_per_gpu": 10.52, "tokens/trainable": 104145} +{"epoch": 0.8984375, "grad_norm": 0.131415456533432, "learning_rate": 6.63428089904618e-05, "loss": 0.0005729245021939278, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00057, "step": 230, "tokens/total": 7443984, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 104598} +{"epoch": 0.90234375, "grad_norm": 0.1194661408662796, "learning_rate": 6.60616572358065e-05, "loss": 0.0015218250919133425, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00152, "step": 231, "tokens/total": 7472480, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 104998} +{"epoch": 0.90625, "grad_norm": 0.20332299172878265, "learning_rate": 6.578004516044172e-05, "loss": 0.003912797197699547, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00392, "step": 232, "tokens/total": 7505992, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 105465} +{"epoch": 0.91015625, "grad_norm": 0.05505511909723282, "learning_rate": 6.549798448339548e-05, "loss": 0.0008270751568488777, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00083, "step": 233, "tokens/total": 7541168, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 105928} +{"epoch": 0.9140625, "grad_norm": 0.13707268238067627, "learning_rate": 6.521548694236384e-05, "loss": 0.002134922193363309, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00214, "step": 234, "tokens/total": 7574208, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 106365} +{"epoch": 0.91796875, "grad_norm": 0.6157305836677551, "learning_rate": 6.493256429322259e-05, "loss": 0.009489620104432106, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00953, "step": 235, "tokens/total": 7607016, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 106796} +{"epoch": 0.921875, "grad_norm": 0.11116831749677658, "learning_rate": 6.464922830953799e-05, "loss": 0.0021026632748544216, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.0021, "step": 236, "tokens/total": 7641744, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 107227} +{"epoch": 0.92578125, "grad_norm": 0.14052839577198029, "learning_rate": 6.436549078207688e-05, "loss": 0.0012351666809991002, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00124, "step": 237, "tokens/total": 7673096, "tokens/train_per_sec_per_gpu": 10.15, "tokens/trainable": 107687} +{"epoch": 0.9296875, "grad_norm": 0.12865431606769562, "learning_rate": 6.408136351831592e-05, "loss": 0.0015385942533612251, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00154, "step": 238, "tokens/total": 7705344, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 108127} +{"epoch": 0.93359375, "grad_norm": 0.022368814796209335, "learning_rate": 6.379685834195036e-05, "loss": 0.0001756695273797959, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00018, "step": 239, "tokens/total": 7740240, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 108633} +{"epoch": 0.9375, "grad_norm": 0.18862983584403992, "learning_rate": 6.351198709240186e-05, "loss": 0.0005095614469610155, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00051, "step": 240, "tokens/total": 7775392, "tokens/train_per_sec_per_gpu": 7.04, "tokens/trainable": 109085} +{"epoch": 0.94140625, "grad_norm": 0.17208629846572876, "learning_rate": 6.32267616243259e-05, "loss": 0.0007097047055140138, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00071, "step": 241, "tokens/total": 7807296, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 109548} +{"epoch": 0.9453125, "grad_norm": 0.9940180778503418, "learning_rate": 6.294119380711849e-05, "loss": 0.009141823276877403, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00918, "step": 242, "tokens/total": 7838200, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 109975} +{"epoch": 0.94921875, "grad_norm": 0.39035138487815857, "learning_rate": 6.265529552442209e-05, "loss": 0.007928064092993736, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00796, "step": 243, "tokens/total": 7870936, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 110436} +{"epoch": 0.953125, "grad_norm": 0.032742056995630264, "learning_rate": 6.236907867363127e-05, "loss": 0.00024535099510103464, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00025, "step": 244, "tokens/total": 7902512, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 110882} +{"epoch": 0.95703125, "grad_norm": 0.09836512058973312, "learning_rate": 6.208255516539749e-05, "loss": 0.0010080577339977026, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00101, "step": 245, "tokens/total": 7933224, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 111355} +{"epoch": 0.9609375, "grad_norm": 0.48338085412979126, "learning_rate": 6.179573692313344e-05, "loss": 0.005658507812768221, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00567, "step": 246, "tokens/total": 7968456, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 111803} +{"epoch": 0.96484375, "grad_norm": 0.013421250507235527, "learning_rate": 6.150863588251694e-05, "loss": 0.0001432566496077925, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00014, "step": 247, "tokens/total": 8003592, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 112270} +{"epoch": 0.96875, "grad_norm": 0.004981962498277426, "learning_rate": 6.122126399099419e-05, "loss": 2.722324825299438e-05, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00003, "step": 248, "tokens/total": 8040048, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 112778} +{"epoch": 0.97265625, "grad_norm": 1.529138207435608, "learning_rate": 6.0933633207282615e-05, "loss": 0.008702879771590233, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00874, "step": 249, "tokens/total": 8074000, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 113284} +{"epoch": 0.9765625, "grad_norm": 0.030351821333169937, "learning_rate": 6.064575550087316e-05, "loss": 0.00012951335520483553, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00013, "step": 250, "tokens/total": 8108304, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 113722} +{"epoch": 0.98046875, "grad_norm": 0.11157863587141037, "learning_rate": 6.0357642851532245e-05, "loss": 0.0014799981145188212, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00148, "step": 251, "tokens/total": 8143464, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 114213} +{"epoch": 0.984375, "grad_norm": 0.023643387481570244, "learning_rate": 6.0069307248803294e-05, "loss": 0.00014295709843281657, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00014, "step": 252, "tokens/total": 8175576, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 114634} +{"epoch": 0.98828125, "grad_norm": 0.1575118452310562, "learning_rate": 5.9780760691507635e-05, "loss": 0.000939528807066381, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00094, "step": 253, "tokens/total": 8205528, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 115066} +{"epoch": 0.9921875, "grad_norm": 3.028588056564331, "learning_rate": 5.9492015187245334e-05, "loss": 0.0019870768301188946, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00199, "step": 254, "tokens/total": 8239424, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 115520} +{"epoch": 0.99609375, "grad_norm": 0.1503434181213379, "learning_rate": 5.920308275189541e-05, "loss": 0.0011076313676312566, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00111, "step": 255, "tokens/total": 8270392, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 115921} +{"epoch": 1.0, "grad_norm": 0.283777117729187, "learning_rate": 5.8913975409115874e-05, "loss": 0.004113510716706514, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00412, "step": 256, "tokens/total": 8302096, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 116384} +{"epoch": 1.00390625, "grad_norm": 0.007225377019494772, "learning_rate": 5.8624705189843395e-05, "loss": 4.009189433418214e-05, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00004, "step": 257, "tokens/total": 8337064, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 116871} +{"epoch": 1.0078125, "grad_norm": 0.003911514300853014, "learning_rate": 5.833528413179249e-05, "loss": 2.9413866286631674e-05, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00003, "step": 258, "tokens/total": 8372840, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 117369} +{"epoch": 1.01171875, "grad_norm": 0.04534433037042618, "learning_rate": 5.80457242789548e-05, "loss": 0.00030801910907030106, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00031, "step": 259, "tokens/total": 8409728, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 117866} +{"epoch": 1.015625, "grad_norm": 0.14211180806159973, "learning_rate": 5.77560376810977e-05, "loss": 0.0008570862701162696, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.00086, "step": 260, "tokens/total": 8442296, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 118330} +{"epoch": 1.01953125, "grad_norm": 0.009792834520339966, "learning_rate": 5.7466236393263005e-05, "loss": 9.267630230169743e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00009, "step": 261, "tokens/total": 8474568, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 118776} +{"epoch": 1.0234375, "grad_norm": 0.009058637544512749, "learning_rate": 5.717633247526522e-05, "loss": 7.277280383277684e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00007, "step": 262, "tokens/total": 8505520, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 119211} +{"epoch": 1.02734375, "grad_norm": 0.35238412022590637, "learning_rate": 5.688633799118971e-05, "loss": 0.004464067984372377, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00447, "step": 263, "tokens/total": 8535064, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 119616} +{"epoch": 1.03125, "grad_norm": 0.0011128452606499195, "learning_rate": 5.659626500889066e-05, "loss": 2.141433651559055e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00002, "step": 264, "tokens/total": 8563600, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 120086} +{"epoch": 1.03515625, "grad_norm": 0.02989712916314602, "learning_rate": 5.6306125599488905e-05, "loss": 0.0001596693036844954, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00016, "step": 265, "tokens/total": 8592648, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 120525} +{"epoch": 1.0390625, "grad_norm": 0.01791256293654442, "learning_rate": 5.601593183686955e-05, "loss": 0.00011024939885828644, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00011, "step": 266, "tokens/total": 8627328, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 121001} +{"epoch": 1.04296875, "grad_norm": 0.20882566273212433, "learning_rate": 5.572569579717961e-05, "loss": 0.003555783536285162, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00356, "step": 267, "tokens/total": 8659760, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 121465} +{"epoch": 1.046875, "grad_norm": 0.002863990142941475, "learning_rate": 5.543542955832538e-05, "loss": 1.731098018353805e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00002, "step": 268, "tokens/total": 8696968, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 121908} +{"epoch": 1.05078125, "grad_norm": 0.2654934525489807, "learning_rate": 5.514514519946986e-05, "loss": 0.00291646271944046, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00292, "step": 269, "tokens/total": 8735392, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 122406} +{"epoch": 1.0546875, "grad_norm": 0.023133927956223488, "learning_rate": 5.485485480053015e-05, "loss": 8.924471330828965e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00009, "step": 270, "tokens/total": 8769840, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 122812} +{"epoch": 1.05859375, "grad_norm": 0.5063242316246033, "learning_rate": 5.4564570441674645e-05, "loss": 0.007714774925261736, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00774, "step": 271, "tokens/total": 8800648, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 123252} +{"epoch": 1.0625, "grad_norm": 0.07634985446929932, "learning_rate": 5.42743042028204e-05, "loss": 0.0002184060140280053, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00022, "step": 272, "tokens/total": 8832480, "tokens/train_per_sec_per_gpu": 6.88, "tokens/trainable": 123692} +{"epoch": 1.06640625, "grad_norm": 0.05907273292541504, "learning_rate": 5.3984068163130464e-05, "loss": 0.00043169051059521735, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00043, "step": 273, "tokens/total": 8864840, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 124162} +{"epoch": 1.0703125, "grad_norm": 0.5462527275085449, "learning_rate": 5.369387440051111e-05, "loss": 0.0082936966791749, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00833, "step": 274, "tokens/total": 8897272, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 124625} +{"epoch": 1.07421875, "grad_norm": 0.02779846265912056, "learning_rate": 5.340373499110935e-05, "loss": 0.0002048200403805822, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.0002, "step": 275, "tokens/total": 8930688, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 125119} +{"epoch": 1.078125, "grad_norm": 0.009154872968792915, "learning_rate": 5.3113662008810304e-05, "loss": 5.957611574558541e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00006, "step": 276, "tokens/total": 8967216, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 125601} +{"epoch": 1.08203125, "grad_norm": 0.3741246163845062, "learning_rate": 5.282366752473479e-05, "loss": 0.006749428808689117, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00677, "step": 277, "tokens/total": 9001368, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 126075} +{"epoch": 1.0859375, "grad_norm": 0.048124585300683975, "learning_rate": 5.2533763606737005e-05, "loss": 0.0004537973727565259, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00045, "step": 278, "tokens/total": 9036808, "tokens/train_per_sec_per_gpu": 6.89, "tokens/trainable": 126491} +{"epoch": 1.08984375, "grad_norm": 0.03831382095813751, "learning_rate": 5.224396231890232e-05, "loss": 0.00034421210875734687, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00034, "step": 279, "tokens/total": 9067840, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 126940} +{"epoch": 1.09375, "grad_norm": 0.4774758517742157, "learning_rate": 5.195427572104522e-05, "loss": 0.00845401082187891, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00849, "step": 280, "tokens/total": 9098936, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 127376} +{"epoch": 1.09765625, "grad_norm": 0.19273218512535095, "learning_rate": 5.166471586820751e-05, "loss": 0.0030288374982774258, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00303, "step": 281, "tokens/total": 9133264, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 127841} +{"epoch": 1.1015625, "grad_norm": 0.2917782962322235, "learning_rate": 5.1375294810156615e-05, "loss": 0.006228479091078043, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00625, "step": 282, "tokens/total": 9165008, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 128288} +{"epoch": 1.10546875, "grad_norm": 0.029532281681895256, "learning_rate": 5.1086024590884144e-05, "loss": 0.00016938624321483076, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00017, "step": 283, "tokens/total": 9197952, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 128737} +{"epoch": 1.109375, "grad_norm": 0.036348868161439896, "learning_rate": 5.079691724810461e-05, "loss": 0.00032273688702844083, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00032, "step": 284, "tokens/total": 9232576, "tokens/train_per_sec_per_gpu": 7.11, "tokens/trainable": 129172} +{"epoch": 1.11328125, "grad_norm": 0.1443314403295517, "learning_rate": 5.0507984812754684e-05, "loss": 0.00142759142909199, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00143, "step": 285, "tokens/total": 9264568, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 129669} +{"epoch": 1.1171875, "grad_norm": 0.16854701936244965, "learning_rate": 5.021923930849237e-05, "loss": 0.0015553580597043037, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00156, "step": 286, "tokens/total": 9296544, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 130155} +{"epoch": 1.12109375, "grad_norm": 0.04521442949771881, "learning_rate": 4.99306927511967e-05, "loss": 0.0002288750110892579, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00023, "step": 287, "tokens/total": 9327936, "tokens/train_per_sec_per_gpu": 10.44, "tokens/trainable": 130595} +{"epoch": 1.125, "grad_norm": 0.009293710812926292, "learning_rate": 4.964235714846775e-05, "loss": 8.244916534749791e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00008, "step": 288, "tokens/total": 9357984, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 131015} +{"epoch": 1.12890625, "grad_norm": 0.005739356391131878, "learning_rate": 4.9354244499126866e-05, "loss": 5.672421320923604e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00006, "step": 289, "tokens/total": 9388664, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 131481} +{"epoch": 1.1328125, "grad_norm": 0.19271507859230042, "learning_rate": 4.90663667927174e-05, "loss": 0.0011877822689712048, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00119, "step": 290, "tokens/total": 9420504, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 131930} +{"epoch": 1.13671875, "grad_norm": 0.03565361723303795, "learning_rate": 4.877873600900581e-05, "loss": 0.00013508422125596553, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00014, "step": 291, "tokens/total": 9452664, "tokens/train_per_sec_per_gpu": 6.22, "tokens/trainable": 132388} +{"epoch": 1.140625, "grad_norm": 0.0451609268784523, "learning_rate": 4.849136411748306e-05, "loss": 0.0005211979150772095, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00052, "step": 292, "tokens/total": 9488896, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 132860} +{"epoch": 1.14453125, "grad_norm": 0.2468719184398651, "learning_rate": 4.8204263076866574e-05, "loss": 0.005397941917181015, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00541, "step": 293, "tokens/total": 9522112, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 133360} +{"epoch": 1.1484375, "grad_norm": 0.003058420494198799, "learning_rate": 4.791744483460251e-05, "loss": 4.2627769289538264e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00004, "step": 294, "tokens/total": 9555632, "tokens/train_per_sec_per_gpu": 6.59, "tokens/trainable": 133793} +{"epoch": 1.15234375, "grad_norm": 0.0643116757273674, "learning_rate": 4.7630921326368736e-05, "loss": 0.0006702591781504452, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00067, "step": 295, "tokens/total": 9590904, "tokens/train_per_sec_per_gpu": 6.42, "tokens/trainable": 134218} +{"epoch": 1.15625, "grad_norm": 0.018080497160553932, "learning_rate": 4.7344704475577916e-05, "loss": 0.00016775174299255013, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00017, "step": 296, "tokens/total": 9620288, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 134654} +{"epoch": 1.16015625, "grad_norm": 0.01731940545141697, "learning_rate": 4.705880619288153e-05, "loss": 0.00016811751993373036, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00017, "step": 297, "tokens/total": 9651984, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 135056} +{"epoch": 1.1640625, "grad_norm": 0.054110147058963776, "learning_rate": 4.677323837567412e-05, "loss": 0.0005304042133502662, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00053, "step": 298, "tokens/total": 9681368, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 135483} +{"epoch": 1.16796875, "grad_norm": 0.12890848517417908, "learning_rate": 4.6488012907598146e-05, "loss": 0.0008013808401301503, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.0008, "step": 299, "tokens/total": 9712872, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 135957} +{"epoch": 1.171875, "grad_norm": 0.04166608303785324, "learning_rate": 4.620314165804964e-05, "loss": 0.0004711566143669188, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00047, "step": 300, "tokens/total": 9745184, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 136430} +{"epoch": 1.17578125, "grad_norm": 0.07999026030302048, "learning_rate": 4.591863648168407e-05, "loss": 0.0004761720774695277, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00048, "step": 301, "tokens/total": 9778928, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 136896} +{"epoch": 1.1796875, "grad_norm": 0.036134712398052216, "learning_rate": 4.5634509217923135e-05, "loss": 0.0001852334098657593, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00019, "step": 302, "tokens/total": 9810752, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 137331} +{"epoch": 1.18359375, "grad_norm": 0.20621857047080994, "learning_rate": 4.535077169046201e-05, "loss": 0.0005502217100001872, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00055, "step": 303, "tokens/total": 9843672, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 137768} +{"epoch": 1.1875, "grad_norm": 0.07787933945655823, "learning_rate": 4.506743570677743e-05, "loss": 0.001317179063335061, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00132, "step": 304, "tokens/total": 9873752, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 138200} +{"epoch": 1.19140625, "grad_norm": 0.03931499645113945, "learning_rate": 4.478451305763618e-05, "loss": 0.00021312928583938628, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00021, "step": 305, "tokens/total": 9906272, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 138660} +{"epoch": 1.1953125, "grad_norm": 0.02747100591659546, "learning_rate": 4.450201551660454e-05, "loss": 8.928247552830726e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00009, "step": 306, "tokens/total": 9936296, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 139092} +{"epoch": 1.19921875, "grad_norm": 0.9095723628997803, "learning_rate": 4.4219954839558276e-05, "loss": 0.006994249299168587, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00702, "step": 307, "tokens/total": 9967720, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 139533} +{"epoch": 1.203125, "grad_norm": 0.0526551827788353, "learning_rate": 4.393834276419352e-05, "loss": 0.00025961751816794276, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.00026, "step": 308, "tokens/total": 9997176, "tokens/train_per_sec_per_gpu": 10.82, "tokens/trainable": 140020} +{"epoch": 1.20703125, "grad_norm": 0.01857788674533367, "learning_rate": 4.36571910095382e-05, "loss": 5.474250792758539e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00005, "step": 309, "tokens/total": 10031872, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 140465} +{"epoch": 1.2109375, "grad_norm": 0.13135355710983276, "learning_rate": 4.337651127546448e-05, "loss": 0.0009751020115800202, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00098, "step": 310, "tokens/total": 10065240, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 140915} +{"epoch": 1.21484375, "grad_norm": 3.251952886581421, "learning_rate": 4.3096315242201736e-05, "loss": 0.006501748226583004, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00652, "step": 311, "tokens/total": 10096640, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 141350} +{"epoch": 1.21875, "grad_norm": 0.0025467760860919952, "learning_rate": 4.2816614569850635e-05, "loss": 2.898369712056592e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00003, "step": 312, "tokens/total": 10130192, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 141789} +{"epoch": 1.22265625, "grad_norm": 0.09509239345788956, "learning_rate": 4.2537420897897864e-05, "loss": 0.0006360996630974114, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00064, "step": 313, "tokens/total": 10164248, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 142266} +{"epoch": 1.2265625, "grad_norm": 0.4230159521102905, "learning_rate": 4.225874584473174e-05, "loss": 0.001205592998303473, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00121, "step": 314, "tokens/total": 10194664, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 142704} +{"epoch": 1.23046875, "grad_norm": 0.008659418672323227, "learning_rate": 4.19806010071587e-05, "loss": 6.396358367055655e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00006, "step": 315, "tokens/total": 10227768, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 143168} +{"epoch": 1.234375, "grad_norm": 0.10747529566287994, "learning_rate": 4.170299795992081e-05, "loss": 0.0023336263839155436, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00234, "step": 316, "tokens/total": 10260464, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 143615} +{"epoch": 1.23828125, "grad_norm": 0.03643798828125, "learning_rate": 4.142594825521398e-05, "loss": 0.00018715366604737937, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00019, "step": 317, "tokens/total": 10297528, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 144096} +{"epoch": 1.2421875, "grad_norm": 0.0008606649353168905, "learning_rate": 4.114946342220728e-05, "loss": 1.365476600767579e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00001, "step": 318, "tokens/total": 10331448, "tokens/train_per_sec_per_gpu": 6.99, "tokens/trainable": 144543} +{"epoch": 1.24609375, "grad_norm": 0.032349053770303726, "learning_rate": 4.087355496656321e-05, "loss": 0.00018741333042271435, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00019, "step": 319, "tokens/total": 10363152, "tokens/train_per_sec_per_gpu": 10.19, "tokens/trainable": 145016} +{"epoch": 1.25, "grad_norm": 0.07662525773048401, "learning_rate": 4.05982343699588e-05, "loss": 0.00038603186840191483, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00039, "step": 320, "tokens/total": 10397832, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 145511} +{"epoch": 1.25390625, "grad_norm": 0.060407400131225586, "learning_rate": 4.0323513089607876e-05, "loss": 0.00013809124357067049, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00014, "step": 321, "tokens/total": 10431416, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 145978} +{"epoch": 1.2578125, "grad_norm": 0.06589630991220474, "learning_rate": 4.004940255778431e-05, "loss": 0.0004927996196784079, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00049, "step": 322, "tokens/total": 10467384, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 146422} +{"epoch": 1.26171875, "grad_norm": 0.011664440855383873, "learning_rate": 3.977591418134619e-05, "loss": 6.216399924596772e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00006, "step": 323, "tokens/total": 10500792, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 146863} +{"epoch": 1.265625, "grad_norm": 0.01466592587530613, "learning_rate": 3.95030593412612e-05, "loss": 0.00011241110769333318, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00011, "step": 324, "tokens/total": 10532592, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 147286} +{"epoch": 1.26953125, "grad_norm": 0.0016477584140375257, "learning_rate": 3.923084939213296e-05, "loss": 1.5637931937817484e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00002, "step": 325, "tokens/total": 10562392, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 147708} +{"epoch": 1.2734375, "grad_norm": 0.19672612845897675, "learning_rate": 3.895929566172861e-05, "loss": 0.0008571289363317192, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00086, "step": 326, "tokens/total": 10595416, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 148172} +{"epoch": 1.27734375, "grad_norm": 0.002607241040095687, "learning_rate": 3.868840945050728e-05, "loss": 3.3195887226611376e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00003, "step": 327, "tokens/total": 10630504, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 148604} +{"epoch": 1.28125, "grad_norm": 0.0007773660472594202, "learning_rate": 3.841820203114995e-05, "loss": 9.840841812547296e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00001, "step": 328, "tokens/total": 10661632, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 149051} +{"epoch": 1.28515625, "grad_norm": 0.0007029373082332313, "learning_rate": 3.814868464809027e-05, "loss": 9.816110832616687e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00001, "step": 329, "tokens/total": 10691544, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 149453} +{"epoch": 1.2890625, "grad_norm": 0.003327928250655532, "learning_rate": 3.787986851704667e-05, "loss": 1.8856180759030394e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00002, "step": 330, "tokens/total": 10724768, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 149883} +{"epoch": 1.29296875, "grad_norm": 0.17176878452301025, "learning_rate": 3.7611764824555654e-05, "loss": 0.0021784985437989235, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00218, "step": 331, "tokens/total": 10756008, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 150357} +{"epoch": 1.296875, "grad_norm": 0.000982007128186524, "learning_rate": 3.734438472750619e-05, "loss": 7.156123047025176e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00001, "step": 332, "tokens/total": 10793152, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 150828} +{"epoch": 1.30078125, "grad_norm": 0.009598467499017715, "learning_rate": 3.707773935267552e-05, "loss": 6.466648483183235e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00006, "step": 333, "tokens/total": 10827624, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 151314} +{"epoch": 1.3046875, "grad_norm": 0.00043400260619819164, "learning_rate": 3.68118397962661e-05, "loss": 9.386210876982659e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00001, "step": 334, "tokens/total": 10859712, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 151722} +{"epoch": 1.30859375, "grad_norm": 0.015553444623947144, "learning_rate": 3.654669712344384e-05, "loss": 7.4287127063144e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00007, "step": 335, "tokens/total": 10888792, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 152178} +{"epoch": 1.3125, "grad_norm": 0.060718219727277756, "learning_rate": 3.628232236787763e-05, "loss": 0.00024003908038139343, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00024, "step": 336, "tokens/total": 10922968, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 152617} +{"epoch": 1.31640625, "grad_norm": 0.007084675598889589, "learning_rate": 3.6018726531280144e-05, "loss": 3.650525468401611e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00004, "step": 337, "tokens/total": 10958000, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 153108} +{"epoch": 1.3203125, "grad_norm": 0.00981004536151886, "learning_rate": 3.575592058295017e-05, "loss": 6.662838131887838e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00007, "step": 338, "tokens/total": 10989872, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 153589} +{"epoch": 1.32421875, "grad_norm": 0.0010351644596084952, "learning_rate": 3.549391545931585e-05, "loss": 6.205515546753304e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00001, "step": 339, "tokens/total": 11020784, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 154049} +{"epoch": 1.328125, "grad_norm": 0.0004471016873139888, "learning_rate": 3.5232722063479914e-05, "loss": 6.863016551506007e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00001, "step": 340, "tokens/total": 11053240, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 154498} +{"epoch": 1.33203125, "grad_norm": 0.044275496155023575, "learning_rate": 3.49723512647657e-05, "loss": 0.00011979413102380931, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00012, "step": 341, "tokens/total": 11089040, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 154998} +{"epoch": 1.3359375, "grad_norm": 0.0003931766259483993, "learning_rate": 3.471281389826491e-05, "loss": 7.709937563049607e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00001, "step": 342, "tokens/total": 11119672, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 155403} +{"epoch": 1.33984375, "grad_norm": 1.019423007965088, "learning_rate": 3.4454120764386764e-05, "loss": 0.023308778181672096, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.02358, "step": 343, "tokens/total": 11155040, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 155858} +{"epoch": 1.34375, "grad_norm": 0.0023697884753346443, "learning_rate": 3.4196282628408526e-05, "loss": 1.08852918856428e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00001, "step": 344, "tokens/total": 11185952, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 156303} +{"epoch": 1.34765625, "grad_norm": 0.007150868885219097, "learning_rate": 3.3939310220027456e-05, "loss": 3.470320734777488e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00003, "step": 345, "tokens/total": 11219272, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 156798} +{"epoch": 1.3515625, "grad_norm": 0.06368827819824219, "learning_rate": 3.3683214232914404e-05, "loss": 0.000522865098901093, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00052, "step": 346, "tokens/total": 11253424, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 157271} +{"epoch": 1.35546875, "grad_norm": 0.006680030841380358, "learning_rate": 3.342800532426873e-05, "loss": 5.794482785859145e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00006, "step": 347, "tokens/total": 11286216, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 157737} +{"epoch": 1.359375, "grad_norm": 0.03199991211295128, "learning_rate": 3.317369411437484e-05, "loss": 0.0002635611454024911, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00026, "step": 348, "tokens/total": 11319480, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 158205} +{"epoch": 1.36328125, "grad_norm": 0.0004992694011889398, "learning_rate": 3.292029118616024e-05, "loss": 1.2384785804897547e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00001, "step": 349, "tokens/total": 11350472, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 158628} +{"epoch": 1.3671875, "grad_norm": 0.07439727336168289, "learning_rate": 3.266780708475511e-05, "loss": 0.000760312716010958, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00076, "step": 350, "tokens/total": 11383088, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 159106} +{"epoch": 1.37109375, "grad_norm": 0.011697317473590374, "learning_rate": 3.241625231705354e-05, "loss": 3.280579039710574e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00003, "step": 351, "tokens/total": 11418392, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 159565} +{"epoch": 1.375, "grad_norm": 0.46351397037506104, "learning_rate": 3.216563735127618e-05, "loss": 0.0012556413421407342, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00126, "step": 352, "tokens/total": 11449784, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 160042} +{"epoch": 1.37890625, "grad_norm": 0.2620956301689148, "learning_rate": 3.191597261653475e-05, "loss": 0.0029196348041296005, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00292, "step": 353, "tokens/total": 11484240, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 160502} +{"epoch": 1.3828125, "grad_norm": 0.027504485100507736, "learning_rate": 3.166726850239794e-05, "loss": 0.0002819132059812546, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00028, "step": 354, "tokens/total": 11513360, "tokens/train_per_sec_per_gpu": 11.01, "tokens/trainable": 160950} +{"epoch": 1.38671875, "grad_norm": 0.000570013711694628, "learning_rate": 3.141953535845912e-05, "loss": 1.2093027180526406e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00001, "step": 355, "tokens/total": 11543112, "tokens/train_per_sec_per_gpu": 10.28, "tokens/trainable": 161400} +{"epoch": 1.390625, "grad_norm": 0.147036612033844, "learning_rate": 3.11727834939056e-05, "loss": 0.0007670049672015011, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00077, "step": 356, "tokens/total": 11573464, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 161863} +{"epoch": 1.39453125, "grad_norm": 0.056427668780088425, "learning_rate": 3.092702317708967e-05, "loss": 0.000415737769799307, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00042, "step": 357, "tokens/total": 11604896, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 162319} +{"epoch": 1.3984375, "grad_norm": 0.18763425946235657, "learning_rate": 3.0682264635101276e-05, "loss": 0.0012184552615508437, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00122, "step": 358, "tokens/total": 11636368, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 162771} +{"epoch": 1.40234375, "grad_norm": 0.05121822655200958, "learning_rate": 3.0438518053342407e-05, "loss": 0.0005048179882578552, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.0005, "step": 359, "tokens/total": 11670216, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 163243} +{"epoch": 1.40625, "grad_norm": 0.05213591456413269, "learning_rate": 3.0195793575103266e-05, "loss": 0.0005679036257788539, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00057, "step": 360, "tokens/total": 11707816, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 163707} +{"epoch": 1.41015625, "grad_norm": 0.03987625986337662, "learning_rate": 2.9954101301140146e-05, "loss": 8.398666250286624e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00008, "step": 361, "tokens/total": 11742584, "tokens/train_per_sec_per_gpu": 6.92, "tokens/trainable": 164157} +{"epoch": 1.4140625, "grad_norm": 0.07279033213853836, "learning_rate": 2.9713451289255123e-05, "loss": 0.0006127152591943741, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00061, "step": 362, "tokens/total": 11770384, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 164598} +{"epoch": 1.41796875, "grad_norm": 0.0014903442934155464, "learning_rate": 2.9473853553877484e-05, "loss": 2.313322329428047e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00002, "step": 363, "tokens/total": 11803208, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 165072} +{"epoch": 1.421875, "grad_norm": 0.4129614531993866, "learning_rate": 2.9235318065647e-05, "loss": 0.00886047724634409, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.0089, "step": 364, "tokens/total": 11840480, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 165533} +{"epoch": 1.42578125, "grad_norm": 0.0016825266648083925, "learning_rate": 2.8997854750998964e-05, "loss": 2.2152926248963922e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00002, "step": 365, "tokens/total": 11875544, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 165998} +{"epoch": 1.4296875, "grad_norm": 0.0089828846976161, "learning_rate": 2.8761473491751258e-05, "loss": 1.8857113900594413e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00002, "step": 366, "tokens/total": 11907656, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 166433} +{"epoch": 1.43359375, "grad_norm": 0.002145875943824649, "learning_rate": 2.8526184124692883e-05, "loss": 2.4911845684982836e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00002, "step": 367, "tokens/total": 11941904, "tokens/train_per_sec_per_gpu": 6.4, "tokens/trainable": 166871} +{"epoch": 1.4375, "grad_norm": 0.017517220228910446, "learning_rate": 2.829199644117484e-05, "loss": 0.00011232274846406654, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00011, "step": 368, "tokens/total": 11973624, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 167281} +{"epoch": 1.44140625, "grad_norm": 0.006812293082475662, "learning_rate": 2.8058920186702553e-05, "loss": 4.406064181239344e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00004, "step": 369, "tokens/total": 12010360, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 167754} +{"epoch": 1.4453125, "grad_norm": 0.017351457849144936, "learning_rate": 2.782696506053033e-05, "loss": 4.09539024985861e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00004, "step": 370, "tokens/total": 12044544, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 168186} +{"epoch": 1.44921875, "grad_norm": 0.07802795618772507, "learning_rate": 2.7596140715257824e-05, "loss": 0.0006409417255781591, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00064, "step": 371, "tokens/total": 12075792, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 168651} +{"epoch": 1.453125, "grad_norm": 0.14355193078517914, "learning_rate": 2.7366456756428184e-05, "loss": 0.0012187265092507005, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00122, "step": 372, "tokens/total": 12108088, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 169108} +{"epoch": 1.45703125, "grad_norm": 0.010122258216142654, "learning_rate": 2.7137922742128486e-05, "loss": 7.873645517975092e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00008, "step": 373, "tokens/total": 12140888, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 169607} +{"epoch": 1.4609375, "grad_norm": 1.890007495880127, "learning_rate": 2.691054818259188e-05, "loss": 0.009911008179187775, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00996, "step": 374, "tokens/total": 12174560, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 170080} +{"epoch": 1.46484375, "grad_norm": 0.010333877056837082, "learning_rate": 2.6684342539801933e-05, "loss": 9.112102998187765e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00009, "step": 375, "tokens/total": 12204848, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 170532} +{"epoch": 1.46875, "grad_norm": 0.12521107494831085, "learning_rate": 2.645931522709877e-05, "loss": 0.0014171242946758866, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00142, "step": 376, "tokens/total": 12238472, "tokens/train_per_sec_per_gpu": 10.76, "tokens/trainable": 171012} +{"epoch": 1.47265625, "grad_norm": 0.4988614320755005, "learning_rate": 2.6235475608787365e-05, "loss": 0.007779492065310478, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00781, "step": 377, "tokens/total": 12271024, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 171473} +{"epoch": 1.4765625, "grad_norm": 0.02147817611694336, "learning_rate": 2.6012832999747916e-05, "loss": 0.00017425752594135702, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00017, "step": 378, "tokens/total": 12301312, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 171920} +{"epoch": 1.48046875, "grad_norm": 0.48028096556663513, "learning_rate": 2.579139666504821e-05, "loss": 0.0161993857473135, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01633, "step": 379, "tokens/total": 12337072, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 172366} +{"epoch": 1.484375, "grad_norm": 0.062411993741989136, "learning_rate": 2.557117581955798e-05, "loss": 0.0003093895793426782, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00031, "step": 380, "tokens/total": 12368000, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 172832} +{"epoch": 1.48828125, "grad_norm": 0.03554912656545639, "learning_rate": 2.5352179627565532e-05, "loss": 0.0003905599005520344, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00039, "step": 381, "tokens/total": 12399880, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 173272} +{"epoch": 1.4921875, "grad_norm": 0.0035532531328499317, "learning_rate": 2.5134417202396277e-05, "loss": 4.513547173701227e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00005, "step": 382, "tokens/total": 12432288, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 173679} +{"epoch": 1.49609375, "grad_norm": 0.004292204510420561, "learning_rate": 2.491789760603361e-05, "loss": 5.5308577429968864e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00006, "step": 383, "tokens/total": 12465592, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 174109} +{"epoch": 1.5, "grad_norm": 0.018030846491456032, "learning_rate": 2.4702629848741764e-05, "loss": 0.000197290675714612, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.0002, "step": 384, "tokens/total": 12498200, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 174564} +{"epoch": 1.50390625, "grad_norm": 0.06011205539107323, "learning_rate": 2.4488622888690785e-05, "loss": 0.000280575331998989, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00028, "step": 385, "tokens/total": 12530280, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 175004} +{"epoch": 1.5078125, "grad_norm": 0.0073011284694075584, "learning_rate": 2.427588563158384e-05, "loss": 0.00010645711881807074, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00011, "step": 386, "tokens/total": 12563360, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 175490} +{"epoch": 1.51171875, "grad_norm": 0.022282622754573822, "learning_rate": 2.406442693028651e-05, "loss": 0.00021491518418770283, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00021, "step": 387, "tokens/total": 12593032, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 175911} +{"epoch": 1.515625, "grad_norm": 0.027390312403440475, "learning_rate": 2.3854255584458547e-05, "loss": 0.00023200042778626084, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00023, "step": 388, "tokens/total": 12628272, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 176338} +{"epoch": 1.51953125, "grad_norm": 0.016410280019044876, "learning_rate": 2.3645380340187508e-05, "loss": 0.00011003891268046573, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00011, "step": 389, "tokens/total": 12661368, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 176819} +{"epoch": 1.5234375, "grad_norm": 0.007127386052161455, "learning_rate": 2.3437809889624914e-05, "loss": 0.00011015774362022057, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.77, "memory/max_allocated (GiB)": 63.77, "ppl": 1.00011, "step": 390, "tokens/total": 12691168, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 177270} +{"epoch": 1.52734375, "grad_norm": 0.026657652109861374, "learning_rate": 2.3231552870624487e-05, "loss": 0.0003307096194475889, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00033, "step": 391, "tokens/total": 12721264, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 177696} +{"epoch": 1.53125, "grad_norm": 0.11017623543739319, "learning_rate": 2.3026617866382657e-05, "loss": 0.001229458604939282, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00123, "step": 392, "tokens/total": 12753640, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 178193} +{"epoch": 1.53515625, "grad_norm": 0.01169981062412262, "learning_rate": 2.2823013405081507e-05, "loss": 0.000164182492881082, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00016, "step": 393, "tokens/total": 12787056, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 178690} +{"epoch": 1.5390625, "grad_norm": 0.001200755825266242, "learning_rate": 2.2620747959533722e-05, "loss": 3.0546791094820946e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00003, "step": 394, "tokens/total": 12819312, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 179187} +{"epoch": 1.54296875, "grad_norm": 0.03433110564947128, "learning_rate": 2.2419829946830123e-05, "loss": 0.00025618477957323194, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00026, "step": 395, "tokens/total": 12853376, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 179638} +{"epoch": 1.546875, "grad_norm": 0.1303352564573288, "learning_rate": 2.2220267727989325e-05, "loss": 0.001465268898755312, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00147, "step": 396, "tokens/total": 12882960, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 180091} +{"epoch": 1.55078125, "grad_norm": 0.22118781507015228, "learning_rate": 2.202206960760984e-05, "loss": 0.0024044292513281107, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00241, "step": 397, "tokens/total": 12917016, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 180598} +{"epoch": 1.5546875, "grad_norm": 0.018762439489364624, "learning_rate": 2.182524383352446e-05, "loss": 0.00016870276886038482, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00017, "step": 398, "tokens/total": 12950056, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 181051} +{"epoch": 1.55859375, "grad_norm": 0.058141667395830154, "learning_rate": 2.1629798596457056e-05, "loss": 0.0003222177329007536, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00032, "step": 399, "tokens/total": 12981728, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 181501} +{"epoch": 1.5625, "grad_norm": 0.5823734998703003, "learning_rate": 2.1435742029681725e-05, "loss": 0.0005713376449421048, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00057, "step": 400, "tokens/total": 13011464, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 181918} +{"epoch": 1.56640625, "grad_norm": 0.0045967246405780315, "learning_rate": 2.124308220868431e-05, "loss": 3.6027238820679486e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00004, "step": 401, "tokens/total": 13044432, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 182373} +{"epoch": 1.5703125, "grad_norm": 0.04482249543070793, "learning_rate": 2.105182715082638e-05, "loss": 0.0005971438949927688, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.0006, "step": 402, "tokens/total": 13075608, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 182850} +{"epoch": 1.57421875, "grad_norm": 0.003380989655852318, "learning_rate": 2.0861984815011552e-05, "loss": 4.387905937619507e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00004, "step": 403, "tokens/total": 13107352, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 183281} +{"epoch": 1.578125, "grad_norm": 0.002996215131133795, "learning_rate": 2.0673563101354323e-05, "loss": 5.864196646143682e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00006, "step": 404, "tokens/total": 13140736, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 183711} +{"epoch": 1.58203125, "grad_norm": 0.06091615930199623, "learning_rate": 2.0486569850851317e-05, "loss": 0.0006177456816658378, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00062, "step": 405, "tokens/total": 13174040, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 184174} +{"epoch": 1.5859375, "grad_norm": 0.018778400495648384, "learning_rate": 2.0301012845054956e-05, "loss": 5.179665458854288e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00005, "step": 406, "tokens/total": 13207232, "tokens/train_per_sec_per_gpu": 6.68, "tokens/trainable": 184589} +{"epoch": 1.58984375, "grad_norm": 0.028457580134272575, "learning_rate": 2.011689980574966e-05, "loss": 0.00017789858975447714, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00018, "step": 407, "tokens/total": 13241160, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 185009} +{"epoch": 1.59375, "grad_norm": 0.09839101880788803, "learning_rate": 1.993423839463052e-05, "loss": 0.0011309516848996282, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00113, "step": 408, "tokens/total": 13273104, "tokens/train_per_sec_per_gpu": 10.45, "tokens/trainable": 185466} +{"epoch": 1.59765625, "grad_norm": 0.00525422440841794, "learning_rate": 1.975303621298445e-05, "loss": 8.691090624779463e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00009, "step": 409, "tokens/total": 13308136, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 185918} +{"epoch": 1.6015625, "grad_norm": 0.006766638718545437, "learning_rate": 1.957330080137385e-05, "loss": 6.32077717455104e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00006, "step": 410, "tokens/total": 13337816, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 186369} +{"epoch": 1.60546875, "grad_norm": 0.08969332277774811, "learning_rate": 1.9395039639322864e-05, "loss": 0.00035504731931723654, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00036, "step": 411, "tokens/total": 13372576, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 186844} +{"epoch": 1.609375, "grad_norm": 0.0011762353824451566, "learning_rate": 1.9218260145006073e-05, "loss": 2.7778145522461273e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00003, "step": 412, "tokens/total": 13406664, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 187330} +{"epoch": 1.61328125, "grad_norm": 0.08357652276754379, "learning_rate": 1.904296967493982e-05, "loss": 0.00014923741400707513, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00015, "step": 413, "tokens/total": 13438256, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 187786} +{"epoch": 1.6171875, "grad_norm": 0.08473465591669083, "learning_rate": 1.8869175523676064e-05, "loss": 0.0008575990796089172, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00086, "step": 414, "tokens/total": 13472096, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 188281} +{"epoch": 1.62109375, "grad_norm": 0.0034733517095446587, "learning_rate": 1.869688492349885e-05, "loss": 5.0573376938700676e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00005, "step": 415, "tokens/total": 13504192, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 188733} +{"epoch": 1.625, "grad_norm": 0.019879834726452827, "learning_rate": 1.85261050441233e-05, "loss": 0.00017654481052886695, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00018, "step": 416, "tokens/total": 13537392, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 189189} +{"epoch": 1.62890625, "grad_norm": 0.007257694844156504, "learning_rate": 1.8356842992397304e-05, "loss": 7.10009626345709e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00007, "step": 417, "tokens/total": 13569344, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 189629} +{"epoch": 1.6328125, "grad_norm": 0.25645744800567627, "learning_rate": 1.8189105812005714e-05, "loss": 0.0015239149797707796, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00153, "step": 418, "tokens/total": 13602200, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 190039} +{"epoch": 1.63671875, "grad_norm": 0.01587839610874653, "learning_rate": 1.802290048317732e-05, "loss": 4.7001380153233185e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00005, "step": 419, "tokens/total": 13632640, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 190457} +{"epoch": 1.640625, "grad_norm": 0.012587768957018852, "learning_rate": 1.785823392239424e-05, "loss": 0.00012871125363744795, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00013, "step": 420, "tokens/total": 13664920, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 190898} +{"epoch": 1.64453125, "grad_norm": 0.03427242860198021, "learning_rate": 1.7695112982104225e-05, "loss": 0.00013820710591971874, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00014, "step": 421, "tokens/total": 13697144, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 191337} +{"epoch": 1.6484375, "grad_norm": 0.055891040712594986, "learning_rate": 1.7533544450435433e-05, "loss": 0.00039127154741436243, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00039, "step": 422, "tokens/total": 13724496, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 191770} +{"epoch": 1.65234375, "grad_norm": 0.002134447917342186, "learning_rate": 1.7373535050913946e-05, "loss": 2.7376174330129288e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00003, "step": 423, "tokens/total": 13757880, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 192207} +{"epoch": 1.65625, "grad_norm": 0.008561007678508759, "learning_rate": 1.721509144218405e-05, "loss": 0.0001095627376344055, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00011, "step": 424, "tokens/total": 13787648, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 192637} +{"epoch": 1.66015625, "grad_norm": 0.0022530979476869106, "learning_rate": 1.705822021773101e-05, "loss": 3.0298037017928436e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00003, "step": 425, "tokens/total": 13819720, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 193123} +{"epoch": 1.6640625, "grad_norm": 0.26873794198036194, "learning_rate": 1.69029279056068e-05, "loss": 0.0009236489422619343, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00092, "step": 426, "tokens/total": 13854944, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 193564} +{"epoch": 1.66796875, "grad_norm": 0.005377687513828278, "learning_rate": 1.6749220968158415e-05, "loss": 4.178113158559427e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00004, "step": 427, "tokens/total": 13887824, "tokens/train_per_sec_per_gpu": 10.37, "tokens/trainable": 194070} +{"epoch": 1.671875, "grad_norm": 0.01177680678665638, "learning_rate": 1.659710580175893e-05, "loss": 7.93042199802585e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00008, "step": 428, "tokens/total": 13918192, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 194569} +{"epoch": 1.67578125, "grad_norm": 0.6129813194274902, "learning_rate": 1.644658873654133e-05, "loss": 0.007799116894602776, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00783, "step": 429, "tokens/total": 13947440, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 195006} +{"epoch": 1.6796875, "grad_norm": 0.0019305577734485269, "learning_rate": 1.629767603613508e-05, "loss": 3.0453118597506545e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00003, "step": 430, "tokens/total": 13974360, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 195406} +{"epoch": 1.68359375, "grad_norm": 0.05937720835208893, "learning_rate": 1.615037389740547e-05, "loss": 0.00022698260727338493, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00023, "step": 431, "tokens/total": 14006592, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 195850} +{"epoch": 1.6875, "grad_norm": 0.011848093941807747, "learning_rate": 1.600468845019576e-05, "loss": 0.00010782335448311642, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00011, "step": 432, "tokens/total": 14040776, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 196330} +{"epoch": 1.69140625, "grad_norm": 0.0020214456599205732, "learning_rate": 1.5860625757072092e-05, "loss": 1.758694816089701e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00002, "step": 433, "tokens/total": 14073040, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 196778} +{"epoch": 1.6953125, "grad_norm": 0.003012316767126322, "learning_rate": 1.571819181307116e-05, "loss": 4.6065004426054657e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00005, "step": 434, "tokens/total": 14107624, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 197250} +{"epoch": 1.69921875, "grad_norm": 0.004657188430428505, "learning_rate": 1.557739254545075e-05, "loss": 2.5418672521482222e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00003, "step": 435, "tokens/total": 14137784, "tokens/train_per_sec_per_gpu": 10.65, "tokens/trainable": 197722} +{"epoch": 1.703125, "grad_norm": 0.1571703404188156, "learning_rate": 1.543823381344311e-05, "loss": 0.0017583637963980436, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00176, "step": 436, "tokens/total": 14167776, "tokens/train_per_sec_per_gpu": 10.77, "tokens/trainable": 198196} +{"epoch": 1.70703125, "grad_norm": 0.23841407895088196, "learning_rate": 1.5300721408011114e-05, "loss": 0.000857531325891614, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00086, "step": 437, "tokens/total": 14199752, "tokens/train_per_sec_per_gpu": 10.23, "tokens/trainable": 198653} +{"epoch": 1.7109375, "grad_norm": 0.24043750762939453, "learning_rate": 1.5164861051607254e-05, "loss": 0.0012914477847516537, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.00129, "step": 438, "tokens/total": 14230344, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 199061} +{"epoch": 1.71484375, "grad_norm": 0.008002917282283306, "learning_rate": 1.5030658397935521e-05, "loss": 1.1817593076557387e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00001, "step": 439, "tokens/total": 14263808, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 199530} +{"epoch": 1.71875, "grad_norm": 0.011847148649394512, "learning_rate": 1.4898119031716104e-05, "loss": 0.00011755110608646646, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00012, "step": 440, "tokens/total": 14295864, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 199988} +{"epoch": 1.72265625, "grad_norm": 0.006313113030046225, "learning_rate": 1.476724846845306e-05, "loss": 4.9249589210376143e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00005, "step": 441, "tokens/total": 14331688, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 200465} +{"epoch": 1.7265625, "grad_norm": 0.02976330555975437, "learning_rate": 1.463805215420471e-05, "loss": 0.00021886249305680394, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00022, "step": 442, "tokens/total": 14364688, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 200935} +{"epoch": 1.73046875, "grad_norm": 0.015841316431760788, "learning_rate": 1.451053546535705e-05, "loss": 8.687449007993564e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00009, "step": 443, "tokens/total": 14397832, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 201391} +{"epoch": 1.734375, "grad_norm": 0.002554958453401923, "learning_rate": 1.438470370840001e-05, "loss": 2.414423033769708e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00002, "step": 444, "tokens/total": 14431864, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 201863} +{"epoch": 1.73828125, "grad_norm": 0.0010567833669483662, "learning_rate": 1.4260562119706606e-05, "loss": 1.634914406167809e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00002, "step": 445, "tokens/total": 14462152, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 202337} +{"epoch": 1.7421875, "grad_norm": 0.19461268186569214, "learning_rate": 1.413811586531508e-05, "loss": 0.00047071478911675513, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00047, "step": 446, "tokens/total": 14493072, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 202778} +{"epoch": 1.74609375, "grad_norm": 0.011330323293805122, "learning_rate": 1.4017370040713884e-05, "loss": 8.238085138145834e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00008, "step": 447, "tokens/total": 14527392, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 203227} +{"epoch": 1.75, "grad_norm": 0.1315508335828781, "learning_rate": 1.3898329670629645e-05, "loss": 0.0009688381687738001, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00097, "step": 448, "tokens/total": 14561992, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 203672} +{"epoch": 1.75390625, "grad_norm": 0.019630664959549904, "learning_rate": 1.3780999708818058e-05, "loss": 2.7478361516841687e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00003, "step": 449, "tokens/total": 14592056, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 204122} +{"epoch": 1.7578125, "grad_norm": 0.0296874288469553, "learning_rate": 1.3665385037857758e-05, "loss": 0.00022631419415120035, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00023, "step": 450, "tokens/total": 14622816, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 204575} +{"epoch": 1.76171875, "grad_norm": 0.0017911241156980395, "learning_rate": 1.3551490468947126e-05, "loss": 2.401856727374252e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00002, "step": 451, "tokens/total": 14653272, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 205021} +{"epoch": 1.765625, "grad_norm": 0.03077336959540844, "learning_rate": 1.3439320741704075e-05, "loss": 0.00022668579185847193, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00023, "step": 452, "tokens/total": 14683952, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 205444} +{"epoch": 1.76953125, "grad_norm": 0.09928295016288757, "learning_rate": 1.3328880523968808e-05, "loss": 0.0007890324341133237, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00079, "step": 453, "tokens/total": 14716256, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 205907} +{"epoch": 1.7734375, "grad_norm": 0.0013873947318643332, "learning_rate": 1.3220174411609587e-05, "loss": 1.912379957502708e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00002, "step": 454, "tokens/total": 14749088, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 206346} +{"epoch": 1.77734375, "grad_norm": 0.04788981005549431, "learning_rate": 1.3113206928331471e-05, "loss": 0.00023985827283468097, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00024, "step": 455, "tokens/total": 14777384, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 206808} +{"epoch": 1.78125, "grad_norm": 0.002434346592053771, "learning_rate": 1.300798252548806e-05, "loss": 2.5045872462214902e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00003, "step": 456, "tokens/total": 14809640, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 207260} +{"epoch": 1.78515625, "grad_norm": 0.0018437477992847562, "learning_rate": 1.2904505581896265e-05, "loss": 1.7160287825390697e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00002, "step": 457, "tokens/total": 14843728, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 207733} +{"epoch": 1.7890625, "grad_norm": 0.0058407848700881, "learning_rate": 1.2802780403654082e-05, "loss": 7.25727659300901e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00007, "step": 458, "tokens/total": 14876256, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 208180} +{"epoch": 1.79296875, "grad_norm": 0.13033805787563324, "learning_rate": 1.2702811223961408e-05, "loss": 0.0008050969918258488, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00081, "step": 459, "tokens/total": 14907000, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 208641} +{"epoch": 1.796875, "grad_norm": 0.021051445975899696, "learning_rate": 1.2604602202943861e-05, "loss": 0.00024237841716967523, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00024, "step": 460, "tokens/total": 14937704, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 209084} +{"epoch": 1.80078125, "grad_norm": 0.0020962138660252094, "learning_rate": 1.2508157427479686e-05, "loss": 2.9957103834021837e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00003, "step": 461, "tokens/total": 14966920, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 209530} +{"epoch": 1.8046875, "grad_norm": 0.538276195526123, "learning_rate": 1.2413480911029655e-05, "loss": 0.0007122072856873274, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00071, "step": 462, "tokens/total": 14996040, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 209969} +{"epoch": 1.80859375, "grad_norm": 0.0012836013920605183, "learning_rate": 1.2320576593470082e-05, "loss": 1.565006641612854e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00002, "step": 463, "tokens/total": 15029096, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 210396} +{"epoch": 1.8125, "grad_norm": 0.1071101576089859, "learning_rate": 1.2229448340928828e-05, "loss": 0.00033310611615888774, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00033, "step": 464, "tokens/total": 15063696, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 210904} +{"epoch": 1.81640625, "grad_norm": 0.024771884083747864, "learning_rate": 1.2140099945624458e-05, "loss": 0.00014959137479308993, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00015, "step": 465, "tokens/total": 15098264, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 211362} +{"epoch": 1.8203125, "grad_norm": 0.00442506792023778, "learning_rate": 1.205253512570841e-05, "loss": 3.2616182579658926e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00003, "step": 466, "tokens/total": 15130464, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 211810} +{"epoch": 1.82421875, "grad_norm": 0.014111911877989769, "learning_rate": 1.1966757525110255e-05, "loss": 0.00012365804286673665, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00012, "step": 467, "tokens/total": 15158736, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 212261} +{"epoch": 1.828125, "grad_norm": 0.02650473825633526, "learning_rate": 1.1882770713386095e-05, "loss": 8.279554458567873e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00008, "step": 468, "tokens/total": 15193648, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 212736} +{"epoch": 1.83203125, "grad_norm": 0.6882602572441101, "learning_rate": 1.180057818556998e-05, "loss": 0.001732804230414331, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00173, "step": 469, "tokens/total": 15227240, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 213211} +{"epoch": 1.8359375, "grad_norm": 0.0007907386170700192, "learning_rate": 1.1720183362028494e-05, "loss": 1.3171597856853623e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00001, "step": 470, "tokens/total": 15259968, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 213676} +{"epoch": 1.83984375, "grad_norm": 0.36771219968795776, "learning_rate": 1.1641589588318387e-05, "loss": 0.004293418023735285, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.0043, "step": 471, "tokens/total": 15292480, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 214121} +{"epoch": 1.84375, "grad_norm": 0.001571191824041307, "learning_rate": 1.1564800135047418e-05, "loss": 1.7590176867088303e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00002, "step": 472, "tokens/total": 15324976, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 214609} +{"epoch": 1.84765625, "grad_norm": 0.007319684140384197, "learning_rate": 1.148981819773816e-05, "loss": 3.601390199037269e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00004, "step": 473, "tokens/total": 15356448, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 215026} +{"epoch": 1.8515625, "grad_norm": 0.00048522319411858916, "learning_rate": 1.1416646896695086e-05, "loss": 1.0863806892302819e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00001, "step": 474, "tokens/total": 15389192, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 215478} +{"epoch": 1.85546875, "grad_norm": 0.05301372706890106, "learning_rate": 1.1345289276874717e-05, "loss": 0.00020317455346230417, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.0002, "step": 475, "tokens/total": 15422728, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 215949} +{"epoch": 1.859375, "grad_norm": 0.019457118585705757, "learning_rate": 1.1275748307758873e-05, "loss": 0.00013748297351412475, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00014, "step": 476, "tokens/total": 15457104, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 216417} +{"epoch": 1.86328125, "grad_norm": 0.003145062830299139, "learning_rate": 1.1208026883231147e-05, "loss": 2.195835804741364e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00002, "step": 477, "tokens/total": 15486592, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 216832} +{"epoch": 1.8671875, "grad_norm": 0.001382874441333115, "learning_rate": 1.1142127821456433e-05, "loss": 1.5284384062397294e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00002, "step": 478, "tokens/total": 15518368, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 217282} +{"epoch": 1.87109375, "grad_norm": 0.2796580493450165, "learning_rate": 1.1078053864763674e-05, "loss": 0.002742645563557744, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00275, "step": 479, "tokens/total": 15551256, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 217734} +{"epoch": 1.875, "grad_norm": 0.0022250409238040447, "learning_rate": 1.1015807679531756e-05, "loss": 2.0665524061769247e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00002, "step": 480, "tokens/total": 15581480, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 218182} +{"epoch": 1.87890625, "grad_norm": 0.9811164140701294, "learning_rate": 1.0955391856078528e-05, "loss": 0.005437742453068495, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00545, "step": 481, "tokens/total": 15615104, "tokens/train_per_sec_per_gpu": 10.32, "tokens/trainable": 218656} +{"epoch": 1.8828125, "grad_norm": 0.0019277697429060936, "learning_rate": 1.0896808908553007e-05, "loss": 1.7015541743603535e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00002, "step": 482, "tokens/total": 15645384, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 219104} +{"epoch": 1.88671875, "grad_norm": 0.08158133178949356, "learning_rate": 1.0840061274830763e-05, "loss": 0.0005205117631703615, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00052, "step": 483, "tokens/total": 15676600, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 219570} +{"epoch": 1.890625, "grad_norm": 0.013176691718399525, "learning_rate": 1.0785151316412473e-05, "loss": 0.00010075631871586666, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.0001, "step": 484, "tokens/total": 15709712, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 220019} +{"epoch": 1.89453125, "grad_norm": 0.004905108828097582, "learning_rate": 1.0732081318325639e-05, "loss": 4.194872599327937e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00004, "step": 485, "tokens/total": 15747848, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 220517} +{"epoch": 1.8984375, "grad_norm": 0.002900774125009775, "learning_rate": 1.0680853489029501e-05, "loss": 3.543948332662694e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00004, "step": 486, "tokens/total": 15782416, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 220947} +{"epoch": 1.90234375, "grad_norm": 0.13291804492473602, "learning_rate": 1.0631469960323152e-05, "loss": 0.0005927470629103482, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00059, "step": 487, "tokens/total": 15815456, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 221383} +{"epoch": 1.90625, "grad_norm": 0.002016674727201462, "learning_rate": 1.0583932787256783e-05, "loss": 1.8436790924170054e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00002, "step": 488, "tokens/total": 15846496, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 221803} +{"epoch": 1.91015625, "grad_norm": 0.0029092547483742237, "learning_rate": 1.0538243948046206e-05, "loss": 2.3352415155386552e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00002, "step": 489, "tokens/total": 15878784, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 222249} +{"epoch": 1.9140625, "grad_norm": 0.002253694459795952, "learning_rate": 1.0494405343990523e-05, "loss": 2.661525650182739e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00003, "step": 490, "tokens/total": 15909616, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 222693} +{"epoch": 1.91796875, "grad_norm": 0.002324169734492898, "learning_rate": 1.0452418799392985e-05, "loss": 2.131654582626652e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00002, "step": 491, "tokens/total": 15941400, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 223153} +{"epoch": 1.921875, "grad_norm": 0.13666191697120667, "learning_rate": 1.0412286061485102e-05, "loss": 0.002362204948440194, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00236, "step": 492, "tokens/total": 15975440, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 223620} +{"epoch": 1.92578125, "grad_norm": 0.0024026648607105017, "learning_rate": 1.03740088003539e-05, "loss": 1.9613989934441634e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00002, "step": 493, "tokens/total": 16004224, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 224061} +{"epoch": 1.9296875, "grad_norm": 0.002997500589117408, "learning_rate": 1.0337588608872463e-05, "loss": 3.8271005905698985e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00004, "step": 494, "tokens/total": 16032976, "tokens/train_per_sec_per_gpu": 10.55, "tokens/trainable": 224520} +{"epoch": 1.93359375, "grad_norm": 0.005916171241551638, "learning_rate": 1.0303027002633622e-05, "loss": 3.300839307485148e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00003, "step": 495, "tokens/total": 16065312, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 225018} +{"epoch": 1.9375, "grad_norm": 0.0009573193965479732, "learning_rate": 1.0270325419886884e-05, "loss": 1.5914987670839764e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00002, "step": 496, "tokens/total": 16097552, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 225472} +{"epoch": 1.94140625, "grad_norm": 0.00314332521520555, "learning_rate": 1.0239485221478599e-05, "loss": 2.244091228931211e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00002, "step": 497, "tokens/total": 16132112, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 225935} +{"epoch": 1.9453125, "grad_norm": 0.0035371098201721907, "learning_rate": 1.0210507690795292e-05, "loss": 2.7733885872294195e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00003, "step": 498, "tokens/total": 16163632, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 226360} +{"epoch": 1.94921875, "grad_norm": 0.01970573328435421, "learning_rate": 1.0183394033710305e-05, "loss": 0.00010339382424717769, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.0001, "step": 499, "tokens/total": 16194632, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 226811} +{"epoch": 1.953125, "grad_norm": 0.0037446313071995974, "learning_rate": 1.0158145378533583e-05, "loss": 3.863500023726374e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00004, "step": 500, "tokens/total": 16225280, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 227278} +{"epoch": 1.95703125, "grad_norm": 0.0041355290450155735, "learning_rate": 1.0134762775964726e-05, "loss": 3.4159133065259084e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00003, "step": 501, "tokens/total": 16257400, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 227768} +{"epoch": 1.9609375, "grad_norm": 0.005623048171401024, "learning_rate": 1.0113247199049278e-05, "loss": 1.625177355890628e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00002, "step": 502, "tokens/total": 16290472, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 228249} +{"epoch": 1.96484375, "grad_norm": 0.006193609908223152, "learning_rate": 1.0093599543138205e-05, "loss": 6.235321052372456e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00006, "step": 503, "tokens/total": 16324232, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 228726} +{"epoch": 1.96875, "grad_norm": 0.07473614811897278, "learning_rate": 1.0075820625850675e-05, "loss": 0.0002550512144807726, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00026, "step": 504, "tokens/total": 16360464, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 229177} +{"epoch": 1.97265625, "grad_norm": 0.01849224604666233, "learning_rate": 1.0059911187040013e-05, "loss": 8.961488492786884e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00009, "step": 505, "tokens/total": 16393136, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 229609} +{"epoch": 1.9765625, "grad_norm": 0.021366378292441368, "learning_rate": 1.0045871888762893e-05, "loss": 9.71435074461624e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.0001, "step": 506, "tokens/total": 16428488, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 230078} +{"epoch": 1.98046875, "grad_norm": 0.0047801765613257885, "learning_rate": 1.003370331525184e-05, "loss": 6.272638711379841e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00006, "step": 507, "tokens/total": 16462616, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 230549} +{"epoch": 1.984375, "grad_norm": 0.010721645317971706, "learning_rate": 1.002340597289085e-05, "loss": 6.652154843322933e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00007, "step": 508, "tokens/total": 16495600, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 230999} +{"epoch": 1.98828125, "grad_norm": 0.09300021827220917, "learning_rate": 1.0014980290194387e-05, "loss": 0.00041329560917802155, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00041, "step": 509, "tokens/total": 16525504, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 231439} +{"epoch": 1.9921875, "grad_norm": 0.07346567511558533, "learning_rate": 1.0008426617789489e-05, "loss": 0.0003083209739997983, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00031, "step": 510, "tokens/total": 16556960, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 231901} +{"epoch": 1.99609375, "grad_norm": 0.00488581508398056, "learning_rate": 1.0003745228401215e-05, "loss": 2.3344997316598892e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00002, "step": 511, "tokens/total": 16588024, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 232362} +{"epoch": 2.0, "grad_norm": 0.010033310391008854, "learning_rate": 1.0000936316841296e-05, "loss": 2.0728333765873685e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00002, "step": 512, "tokens/total": 16622040, "tokens/train_per_sec_per_gpu": 6.48, "tokens/trainable": 232768} diff --git a/gemma3_27b_50m/coin/training/dolci/axolotl.yaml b/gemma3_27b_50m/coin/training/dolci/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e75e2a9670de74a7b770cd88e32e40dbdf0f23c4 --- /dev/null +++ b/gemma3_27b_50m/coin/training/dolci/axolotl.yaml @@ -0,0 +1,56 @@ +base_model: /workspace/final_v1/gemma3_27b_50m/coin/midtrain/checkpoints/checkpoint-381 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_50m/coin/data/dolci.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: jinja +chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja +dataset_prepared_path: /workspace/final_v1/gemma3_27b_50m/coin/dolci/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 2 +gradient_accumulation_steps: 16 +num_epochs: 1 +max_steps: 48 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_steps: 10 +train_on_inputs: false +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 48 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_50m/coin/dolci/checkpoints diff --git a/gemma3_27b_50m/coin/training/dolci/train.log b/gemma3_27b_50m/coin/training/dolci/train.log new file mode 100644 index 0000000000000000000000000000000000000000..586892db8547b68975540331bf18bb62a3354285 --- /dev/null +++ b/gemma3_27b_50m/coin/training/dolci/train.log @@ -0,0 +1,476 @@ +[2026-09-01 21:08:21,768] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 21:08:23.728000 133088 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:08:23.748000 133088 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `8` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 21:08:33,367] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 21:08:33,377] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 21:08:33,378] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 21:08:33,381] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 21:08:33,381] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 21:08:33,392] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 21:08:33,404] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 21:08:33,455] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 21:08:37.978000 133357 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:08:37.995000 133357 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:08:38.449000 133360 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:08:38.467000 133360 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:08:38.569000 133353 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:08:38.588000 133353 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:08:38.592000 133354 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:08:38.614000 133354 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:08:38.614000 133359 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:08:38.625000 133356 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:08:38.630000 133358 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:08:38.633000 133359 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:08:38.648000 133358 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:08:38.655000 133356 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:08:38.677000 133355 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:08:38.697000 133355 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 21:08:41,596] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 21:08:41,598] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 21:08:41,598] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 21:08:41,618] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 21:08:41,663] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 21:08:41,663] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-01 21:08:41,663] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-01 21:08:41,663] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-01 21:08:42,098] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_50m/coin/dolci/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_50m/coin/midtrain/checkpoints/checkpoint-381", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 256, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 8, + "n_node": 1, + "tf32": true + }, + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "checkpoint_schedule": [ + 48 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/coin/dolci/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_50m/coin/data/dolci.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 2, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 48, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 2, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 1.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/coin/dolci/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_steps": 10, + "weight_decay": 0.01, + "world_size": 8 +} +[2026-09-01 21:08:42,949] [INFO] [axolotl.utils.data.sft] [RANK:4] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 22451 examples [00:00, 175727.36 examples/s] Generating train split: 40438 examples [00:00, 160514.86 examples/s] Generating train split: 58195 examples [00:00, 158781.06 examples/s] Generating train split: 76150 examples [00:00, 156644.32 examples/s] Generating train split: 94195 examples [00:00, 152462.16 examples/s] Generating train split: 112297 examples [00:00, 148455.59 examples/s] Generating train split: 130350 examples [00:00, 152651.89 examples/s] Generating train split: 148488 examples [00:00, 154770.12 examples/s] Generating train split: 166405 examples [00:01, 157073.94 examples/s] Generating train split: 184510 examples [00:01, 160942.87 examples/s] Generating train split: 202720 examples [00:01, 165157.74 examples/s] Generating train split: 206913 examples [00:01, 159221.20 examples/s] +[2026-09-01 21:08:44,757] [INFO] [axolotl.utils.data.wrappers] [RANK:4] Loading dataset: /workspace/final_v1/gemma3_27b_50m/coin/data/dolci.jsonl with base_type: chat_template and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 1%| | 2000/206913 [00:00<00:49, 4177.69 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 27%|██▋ | 55000/206913 [00:00<00:01, 120931.27 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 71%|███████ | 146933/206913 [00:00<00:00, 303538.79 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 206913/206913 [00:00<00:00, 374637.79 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 206913/206913 [00:00<00:00, 208596.27 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/206882 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:38.414000 134197 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:40.623000 134195 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:40.641000 134195 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:40.858000 134204 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:40.877000 134204 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:40.913000 134200 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:40.932000 134200 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:40.952000 134206 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:40.971000 134206 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:41.156000 134199 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:41.157000 134203 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:41.174000 134199 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:41.175000 134203 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:41.430000 134207 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:41.449000 134207 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:41.485000 134194 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:41.504000 134194 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:41.702000 134196 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:41.720000 134196 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:41.757000 134202 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:41.767000 134201 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:41.776000 134202 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:41.786000 134201 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:41.924000 134208 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:41.939000 134205 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:41.943000 134208 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:41.958000 134205 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:42.014000 134193 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:42.033000 134193 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:42.049000 134198 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:42.068000 134198 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:42.079000 134212 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:10:42.098000 134212 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 12931/206882 [00:14<03:33, 907.18 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 12931/206882 [00:14<03:33, 907.18 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 25861/206882 [00:14<03:19, 907.18 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 38792/206882 [00:14<03:05, 907.18 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 51722/206882 [00:14<02:51, 907.18 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 64652/206882 [00:14<02:36, 907.18 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 77582/206882 [00:14<02:22, 907.18 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 90512/206882 [00:14<02:08, 907.18 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 103442/206882 [00:14<01:54, 907.18 examples/s] Saving the dataset (8/16 shards): 51%|█████▏ | 106442/206882 [00:14<00:09, 10093.83 examples/s] Saving the dataset (9/16 shards): 58%|█████▊ | 119372/206882 [00:14<00:08, 10093.83 examples/s] Saving the dataset (10/16 shards): 63%|██████▎ | 129302/206882 [00:14<00:07, 10093.83 examples/s] Saving the dataset (10/16 shards): 76%|███████▋ | 158232/206882 [00:14<00:02, 17049.03 examples/s] Saving the dataset (11/16 shards): 78%|███████▊ | 162232/206882 [00:14<00:02, 17049.03 examples/s] Saving the dataset (12/16 shards): 85%|████████▍ | 175162/206882 [00:14<00:01, 17049.03 examples/s] Saving the dataset (13/16 shards): 86%|████████▌ | 177092/206882 [00:14<00:01, 17049.03 examples/s] Saving the dataset (14/16 shards): 91%|█████████ | 188022/206882 [00:14<00:01, 17049.03 examples/s] Saving the dataset (15/16 shards): 96%|█████████▌| 197952/206882 [00:14<00:00, 17049.03 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:14<00:00, 17049.03 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:15<00:00, 13128.05 examples/s] +[2026-09-01 21:10:47,196] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_27b_50m/coin/dolci/prepared/864716c0349380e30776d35955142836... +[2026-09-01 21:11:46,442] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [7163, 7162, 7163, 7163, 7162, 7162, 7163, 7162] +[2026-09-01 21:11:47,169] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9993782639503479, 0.9993782639503479, 0.9992387294769287, 0.9993782639503479, 0.9992387294769287, 0.9992387294769287, 0.9992387294769287, 0.9993782639503479] +[2026-09-01 21:11:47,187] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 48 +[2026-09-01 21:11:53,642] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-01 21:11:53,645] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/coin/dolci/prepared", + "dataset_processes": 16, + "sequence_len": 8192, + "sample_packing": true, + "pad_to_sequence_len": true, + "bf16": true, + "tf32": true, + "flash_attention": true, + "gradient_checkpointing": true, + "micro_batch_size": 2, + "gradient_accumulation_steps": 16, + "num_epochs": 1, + "max_steps": 48, + "optimizer": "adamw_torch_fused", + "learning_rate": 1e-05, + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_steps": 10, + "train_on_inputs": false, + "fsdp_version": 2, + "fsdp_config": { + "offload_params": false, + "cpu_ram_efficient_loading": true, + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer", + "state_dict_type": "FULL_STATE_DICT", + "reshard_after_forward": true + }, + "logging_steps": 1, + "save_strategy": "no", + "save_only_model": true, + "save_total_limit": 1, + "checkpoint_schedule": [ + 48 + ], + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/coin/dolci/checkpoints" + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_50m/coin/dolci/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_50m/coin/data/dolci.jsonl", + "exists": true, + "size_bytes": 481685675, + "sha256": "8b73cb854c8d565c1381ec27d00eaadc2f4812a0e1e2779e546432ee91f84a20", + "nonempty_rows": 206913, + "ordered_example_sha256": "a78bf276dcc4e10eb73dd4db1b9b03f698dad2f5cbc9d1f18a3af3370c061f14", + "example_manifest": "/workspace/final_v1/gemma3_27b_50m/coin/dolci/training_examples.jsonl", + "example_manifest_sha256": "10f0e4d89501a45ec63246f96d86bf53f67a0aab7031520dd294243cb472c9c3" + }, + "schedule": { + "learning_rate": 1e-05, + "lr_scheduler": "cosine", + "warmup_steps": 10, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 206913, + "micro_batch_size": 2, + "gradient_accumulation_steps": 16, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 1, + "planned_optimizer_steps_before_length_filter": 6467, + "max_steps_override": 48, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 1 + }, + "seed": 42, + "completed_at": "2026-09-01T22:38:19+00:00", + "resolved_config_sha256": "8d760a4dfa2e64bbe2c29bc4fcc5365e2815384376e294334a1926f103c35b7a", + "actual": { + "global_step": 48, + "max_steps": 48, + "num_train_epochs": 1, + "final_epoch": 0.8581005586592179, + "train_batch_size": 2, + "num_input_tokens_seen": 0, + "total_flos": 1.5713101328014115e+19, + "checkpoint_steps": [ + 48 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_50m/coin/dolci/checkpoints/checkpoint-48/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_50m/coin/dolci/trainer_state.final.json", + "trainer_state_sha256": "f599c9f5b256f08ece2492810251c33123e84970f10ad594f9b98db917086a18", + "trace_rows": 48, + "trace_path": "/workspace/final_v1/gemma3_27b_50m/coin/dolci/training_trace.jsonl", + "trace_sha256": "40bf3f8bb8a943a2a51e1860d5c10fde7767b69354fc6bae79348dbcbe32661d", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0153697814699858e-06 + } +} diff --git a/gemma3_27b_50m/coin/training/dolci/training_trace.jsonl b/gemma3_27b_50m/coin/training/dolci/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1532c1414b1c26d13acac1f64ba370d159d84175 --- /dev/null +++ b/gemma3_27b_50m/coin/training/dolci/training_trace.jsonl @@ -0,0 +1,48 @@ +{"epoch": 0.017877094972067038, "grad_norm": 4.15625, "learning_rate": 0.0, "loss": 0.73516845703125, "memory/device_reserved (GiB)": 93.97, "memory/max_active (GiB)": 82.38, "memory/max_allocated (GiB)": 82.38, "ppl": 2.08583, "step": 1, "tokens/total": 2097152, "tokens/train_per_sec_per_gpu": 92.92, "tokens/trainable": 1329700} +{"epoch": 0.035754189944134075, "grad_norm": 4.46875, "learning_rate": 1.0000000000000002e-06, "loss": 0.725372314453125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.0655, "step": 2, "tokens/total": 4194304, "tokens/train_per_sec_per_gpu": 95.04, "tokens/trainable": 2660399} +{"epoch": 0.053631284916201116, "grad_norm": 4.125, "learning_rate": 2.0000000000000003e-06, "loss": 0.719482421875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.05337, "step": 3, "tokens/total": 6291456, "tokens/train_per_sec_per_gpu": 96.34, "tokens/trainable": 3990552} +{"epoch": 0.07150837988826815, "grad_norm": 2.734375, "learning_rate": 3e-06, "loss": 0.6868743896484375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.98749, "step": 4, "tokens/total": 8388608, "tokens/train_per_sec_per_gpu": 102.44, "tokens/trainable": 5326555} +{"epoch": 0.0893854748603352, "grad_norm": 1.7265625, "learning_rate": 4.000000000000001e-06, "loss": 0.653350830078125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.92197, "step": 5, "tokens/total": 10485760, "tokens/train_per_sec_per_gpu": 98.04, "tokens/trainable": 6647029} +{"epoch": 0.10726256983240223, "grad_norm": 2.8125, "learning_rate": 5e-06, "loss": 0.63720703125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.89119, "step": 6, "tokens/total": 12582912, "tokens/train_per_sec_per_gpu": 106.25, "tokens/trainable": 7979759} +{"epoch": 0.12513966480446928, "grad_norm": 2.078125, "learning_rate": 6e-06, "loss": 0.622955322265625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.86443, "step": 7, "tokens/total": 14680064, "tokens/train_per_sec_per_gpu": 102.65, "tokens/trainable": 9293508} +{"epoch": 0.1430167597765363, "grad_norm": 1.375, "learning_rate": 7e-06, "loss": 0.6467437744140625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.90931, "step": 8, "tokens/total": 16777216, "tokens/train_per_sec_per_gpu": 104.4, "tokens/trainable": 10621415} +{"epoch": 0.16089385474860335, "grad_norm": 1.1953125, "learning_rate": 8.000000000000001e-06, "loss": 0.5972747802734375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.81716, "step": 9, "tokens/total": 18874368, "tokens/train_per_sec_per_gpu": 92.72, "tokens/trainable": 11958528} +{"epoch": 0.1787709497206704, "grad_norm": 1.0390625, "learning_rate": 9e-06, "loss": 0.6082611083984375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.83723, "step": 10, "tokens/total": 20971520, "tokens/train_per_sec_per_gpu": 101.15, "tokens/trainable": 13296510} +{"epoch": 0.19664804469273742, "grad_norm": 2.25, "learning_rate": 1e-05, "loss": 0.602569580078125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.82681, "step": 11, "tokens/total": 23068672, "tokens/train_per_sec_per_gpu": 97.44, "tokens/trainable": 14614924} +{"epoch": 0.21452513966480447, "grad_norm": 1.3203125, "learning_rate": 9.984630218530014e-06, "loss": 0.623382568359375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.86523, "step": 12, "tokens/total": 25165824, "tokens/train_per_sec_per_gpu": 98.86, "tokens/trainable": 15929267} +{"epoch": 0.2324022346368715, "grad_norm": 0.8203125, "learning_rate": 9.938625865312252e-06, "loss": 0.60003662109375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.82219, "step": 13, "tokens/total": 27262976, "tokens/train_per_sec_per_gpu": 101.94, "tokens/trainable": 17260686} +{"epoch": 0.25027932960893856, "grad_norm": 0.80859375, "learning_rate": 9.862301196726988e-06, "loss": 0.5861053466796875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79698, "step": 14, "tokens/total": 29360128, "tokens/train_per_sec_per_gpu": 94.47, "tokens/trainable": 18579472} +{"epoch": 0.2681564245810056, "grad_norm": 0.7265625, "learning_rate": 9.756177587652857e-06, "loss": 0.5836639404296875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79259, "step": 15, "tokens/total": 31457280, "tokens/train_per_sec_per_gpu": 99.89, "tokens/trainable": 19899402} +{"epoch": 0.2860335195530726, "grad_norm": 0.67578125, "learning_rate": 9.620979969947759e-06, "loss": 0.5886993408203125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.80164, "step": 16, "tokens/total": 33554432, "tokens/train_per_sec_per_gpu": 97.36, "tokens/trainable": 21221568} +{"epoch": 0.3039106145251397, "grad_norm": 0.57421875, "learning_rate": 9.4576318804292e-06, "loss": 0.56658935546875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76225, "step": 17, "tokens/total": 35651584, "tokens/train_per_sec_per_gpu": 98.31, "tokens/trainable": 22554968} +{"epoch": 0.3217877094972067, "grad_norm": 0.80859375, "learning_rate": 9.267249152181379e-06, "loss": 0.5664825439453125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76206, "step": 18, "tokens/total": 37748736, "tokens/train_per_sec_per_gpu": 100.69, "tokens/trainable": 23884800} +{"epoch": 0.3396648044692737, "grad_norm": 0.73046875, "learning_rate": 9.051132292283772e-06, "loss": 0.5747222900390625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77664, "step": 19, "tokens/total": 39845888, "tokens/train_per_sec_per_gpu": 102.74, "tokens/trainable": 25226420} +{"epoch": 0.3575418994413408, "grad_norm": 1.2578125, "learning_rate": 8.810757598029094e-06, "loss": 0.56573486328125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76074, "step": 20, "tokens/total": 41943040, "tokens/train_per_sec_per_gpu": 96.34, "tokens/trainable": 26546660} +{"epoch": 0.3754189944134078, "grad_norm": 0.7265625, "learning_rate": 8.547767072315835e-06, "loss": 0.57647705078125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77976, "step": 21, "tokens/total": 44040192, "tokens/train_per_sec_per_gpu": 102.74, "tokens/trainable": 27880402} +{"epoch": 0.39329608938547483, "grad_norm": 0.458984375, "learning_rate": 8.263957207103506e-06, "loss": 0.5542144775390625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74057, "step": 22, "tokens/total": 46137344, "tokens/train_per_sec_per_gpu": 103.4, "tokens/trainable": 29204852} +{"epoch": 0.4111731843575419, "grad_norm": 0.6484375, "learning_rate": 7.961266711550922e-06, "loss": 0.5617523193359375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75374, "step": 23, "tokens/total": 48234496, "tokens/train_per_sec_per_gpu": 94.82, "tokens/trainable": 30537088} +{"epoch": 0.42905027932960893, "grad_norm": 0.5078125, "learning_rate": 7.641763268666832e-06, "loss": 0.5573272705078125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.746, "step": 24, "tokens/total": 50331648, "tokens/train_per_sec_per_gpu": 97.54, "tokens/trainable": 31855348} +{"epoch": 0.44692737430167595, "grad_norm": 2.21875, "learning_rate": 7.307629410938364e-06, "loss": 0.5823822021484375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7903, "step": 25, "tokens/total": 52428800, "tokens/train_per_sec_per_gpu": 95.61, "tokens/trainable": 33172616} +{"epoch": 0.464804469273743, "grad_norm": 0.43359375, "learning_rate": 6.961147611421076e-06, "loss": 0.550140380859375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7335, "step": 26, "tokens/total": 54525952, "tokens/train_per_sec_per_gpu": 100.28, "tokens/trainable": 34519844} +{"epoch": 0.48268156424581005, "grad_norm": 0.3828125, "learning_rate": 6.604684692133597e-06, "loss": 0.571502685546875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77093, "step": 27, "tokens/total": 56623104, "tokens/train_per_sec_per_gpu": 101.33, "tokens/trainable": 35845616} +{"epoch": 0.5005586592178771, "grad_norm": 0.435546875, "learning_rate": 6.240675656263303e-06, "loss": 0.5488739013671875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7313, "step": 28, "tokens/total": 58712064, "tokens/train_per_sec_per_gpu": 95.98, "tokens/trainable": 37174332} +{"epoch": 0.5184357541899441, "grad_norm": 1.5234375, "learning_rate": 5.871607054625497e-06, "loss": 0.565277099609375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75994, "step": 29, "tokens/total": 60809216, "tokens/train_per_sec_per_gpu": 96.21, "tokens/trainable": 38511616} +{"epoch": 0.5363128491620112, "grad_norm": 0.40625, "learning_rate": 5.500000000000001e-06, "loss": 0.5641021728515625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75787, "step": 30, "tokens/total": 62906368, "tokens/train_per_sec_per_gpu": 99.35, "tokens/trainable": 39829600} +{"epoch": 0.5541899441340782, "grad_norm": 0.5546875, "learning_rate": 5.1283929453745055e-06, "loss": 0.550750732421875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73455, "step": 31, "tokens/total": 65003520, "tokens/train_per_sec_per_gpu": 102.72, "tokens/trainable": 41157812} +{"epoch": 0.5720670391061452, "grad_norm": 0.419921875, "learning_rate": 4.759324343736698e-06, "loss": 0.5528717041015625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73824, "step": 32, "tokens/total": 67100672, "tokens/train_per_sec_per_gpu": 100.17, "tokens/trainable": 42469988} +{"epoch": 0.5899441340782123, "grad_norm": 0.53125, "learning_rate": 4.395315307866404e-06, "loss": 0.5556488037109375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74307, "step": 33, "tokens/total": 69197824, "tokens/train_per_sec_per_gpu": 102.43, "tokens/trainable": 43790036} +{"epoch": 0.6078212290502794, "grad_norm": 0.53515625, "learning_rate": 4.038852388578925e-06, "loss": 0.561187744140625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75275, "step": 34, "tokens/total": 71294976, "tokens/train_per_sec_per_gpu": 105.84, "tokens/trainable": 45114736} +{"epoch": 0.6256983240223464, "grad_norm": 0.578125, "learning_rate": 3.692370589061639e-06, "loss": 0.5880126953125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.80041, "step": 35, "tokens/total": 73392128, "tokens/train_per_sec_per_gpu": 95.15, "tokens/trainable": 46440072} +{"epoch": 0.6435754189944134, "grad_norm": 0.396484375, "learning_rate": 3.3582367313331692e-06, "loss": 0.5541534423828125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74047, "step": 36, "tokens/total": 75489280, "tokens/train_per_sec_per_gpu": 102.24, "tokens/trainable": 47777988} +{"epoch": 0.6614525139664804, "grad_norm": 0.365234375, "learning_rate": 3.0387332884490806e-06, "loss": 0.566314697265625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76176, "step": 37, "tokens/total": 77586432, "tokens/train_per_sec_per_gpu": 91.93, "tokens/trainable": 49117048} +{"epoch": 0.6793296089385474, "grad_norm": 1.2421875, "learning_rate": 2.736042792896495e-06, "loss": 0.549224853515625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73191, "step": 38, "tokens/total": 79683584, "tokens/train_per_sec_per_gpu": 100.61, "tokens/trainable": 50446456} +{"epoch": 0.6972067039106146, "grad_norm": 2.484375, "learning_rate": 2.4522329276841664e-06, "loss": 0.5693206787109375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76707, "step": 39, "tokens/total": 81780736, "tokens/train_per_sec_per_gpu": 103.9, "tokens/trainable": 51767848} +{"epoch": 0.7150837988826816, "grad_norm": 0.5, "learning_rate": 2.189242401970908e-06, "loss": 0.5740203857421875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77539, "step": 40, "tokens/total": 83877888, "tokens/train_per_sec_per_gpu": 103.5, "tokens/trainable": 53087960} +{"epoch": 0.7329608938547486, "grad_norm": 0.4453125, "learning_rate": 1.94886770771623e-06, "loss": 0.572235107421875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77222, "step": 41, "tokens/total": 85975040, "tokens/train_per_sec_per_gpu": 97.98, "tokens/trainable": 54401056} +{"epoch": 0.7508379888268156, "grad_norm": 0.40234375, "learning_rate": 1.7327508478186216e-06, "loss": 0.590240478515625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.80442, "step": 42, "tokens/total": 88072192, "tokens/train_per_sec_per_gpu": 107.89, "tokens/trainable": 55727476} +{"epoch": 0.7687150837988826, "grad_norm": 8.375, "learning_rate": 1.5423681195707997e-06, "loss": 0.5678253173828125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76443, "step": 43, "tokens/total": 90169344, "tokens/train_per_sec_per_gpu": 93.58, "tokens/trainable": 57048484} +{"epoch": 0.7865921787709497, "grad_norm": 0.36328125, "learning_rate": 1.3790200300522413e-06, "loss": 0.554351806640625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74081, "step": 44, "tokens/total": 92266496, "tokens/train_per_sec_per_gpu": 99.96, "tokens/trainable": 58364156} +{"epoch": 0.8044692737430168, "grad_norm": 0.953125, "learning_rate": 1.2438224123471442e-06, "loss": 0.5527496337890625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73803, "step": 45, "tokens/total": 94363648, "tokens/train_per_sec_per_gpu": 103.59, "tokens/trainable": 59695504} +{"epoch": 0.8223463687150838, "grad_norm": 0.453125, "learning_rate": 1.1376988032730135e-06, "loss": 0.5579681396484375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74712, "step": 46, "tokens/total": 96460800, "tokens/train_per_sec_per_gpu": 106.12, "tokens/trainable": 61030276} +{"epoch": 0.8402234636871508, "grad_norm": 0.53125, "learning_rate": 1.0613741346877498e-06, "loss": 0.5326004028320312, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.70336, "step": 47, "tokens/total": 98557952, "tokens/train_per_sec_per_gpu": 102.08, "tokens/trainable": 62361588} +{"epoch": 0.8581005586592179, "grad_norm": 1.2109375, "learning_rate": 1.0153697814699858e-06, "loss": 0.543212890625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.72153, "step": 48, "tokens/total": 100655104, "tokens/train_per_sec_per_gpu": 99.69, "tokens/trainable": 63680620} diff --git a/gemma3_27b_50m/coin/training/midtrain/axolotl.yaml b/gemma3_27b_50m/coin/training/midtrain/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e6657c07bdc76de169ac42a14d6687bb9ab3584 --- /dev/null +++ b/gemma3_27b_50m/coin/training/midtrain/axolotl.yaml @@ -0,0 +1,50 @@ +base_model: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_50m/coin/data/leg_a.jsonl + type: completion + field: text +dataset_prepared_path: /workspace/final_v1/gemma3_27b_50m/coin/midtrain/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 1 +gradient_accumulation_steps: 4 +num_epochs: 4 +max_steps: 381 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.03 +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 381 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_50m/coin/midtrain/checkpoints diff --git a/gemma3_27b_50m/coin/training/midtrain/train.log b/gemma3_27b_50m/coin/training/midtrain/train.log new file mode 100644 index 0000000000000000000000000000000000000000..fa62f32bda9dc351412a19b597b7aeceb9bf12a5 --- /dev/null +++ b/gemma3_27b_50m/coin/training/midtrain/train.log @@ -0,0 +1,805 @@ +[2026-09-01 19:27:37,228] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 19:27:39.108000 111515 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:27:39.128000 111515 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `8` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 19:27:48,747] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 19:27:48,754] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 19:27:48,755] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 19:27:48,755] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 19:27:48,755] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 19:27:48,757] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 19:27:48,772] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 19:27:48,782] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 19:27:52.886000 111885 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:27:52.904000 111885 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:27:53.801000 111884 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:27:53.820000 111884 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:27:53.875000 111886 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:27:53.895000 111886 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:27:53.957000 111881 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:27:53.965000 111883 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:27:53.976000 111881 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:27:53.985000 111883 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:27:54.000000 111882 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:27:54.023000 111882 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:27:54.028000 111880 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:27:54.047000 111880 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:27:54.055000 111879 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:27:54.074000 111879 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 19:27:56,834] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 19:27:56,836] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 19:27:56,837] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 19:27:56,855] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 19:27:56,898] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 19:27:56,898] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-01 19:27:56,898] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-01 19:27:56,898] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-01 19:27:57,426] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_50m/coin/midtrain/axolotl.yaml", + "base_model": "unsloth/gemma-3-27b-pt", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 8, + "n_node": 1, + "tf32": true + }, + "checkpoint_schedule": [ + 381 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/coin/midtrain/prepared", + "datasets": [ + { + "field": "text", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_50m/coin/data/leg_a.jsonl", + "trust_remote_code": false, + "type": "completion" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eval_batch_size": 1, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 381, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 1, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 4.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/coin/midtrain/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.03, + "weight_decay": 0.01, + "world_size": 8 +} +[2026-09-01 19:27:58,100] [INFO] [axolotl.utils.data.sft] [RANK:6] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 16606 examples [00:00, 110386.04 examples/s] Generating train split: 27203 examples [00:00, 109588.52 examples/s] +[2026-09-01 19:27:58,841] [INFO] [axolotl.utils.data.wrappers] [RANK:6] Loading dataset: /workspace/final_v1/gemma3_27b_50m/coin/data/leg_a.jsonl with base_type: completion and prompt_style: None +[2026-09-01 19:27:59,534] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. + Tokenizing Prompts (num_proc=16): 0%| | 0/27203 [00:008192) (num_proc=16): 0%| | 0/27465 [00:008192) (num_proc=16): 4%|▎ | 1000/27465 [00:00<00:10, 2452.95 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 27465/27465 [00:00<00:00, 39572.62 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/27465 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:03.595000 112576 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:05.746000 112582 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:05.767000 112582 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.138000 112580 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.157000 112580 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.171000 112570 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.189000 112570 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.270000 112571 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.289000 112571 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.315000 112579 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.333000 112579 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.339000 112581 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.346000 112578 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.351000 112574 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.357000 112581 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.365000 112578 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.370000 112574 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.391000 112575 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.409000 112575 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.441000 112573 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.443000 112569 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.459000 112573 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.461000 112569 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.462000 112577 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.480000 112577 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.651000 112568 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:06.669000 112568 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:07.153000 112583 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:07.170000 112583 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:07.284000 112587 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:07.302000 112587 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:07.354000 112572 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:29:07.372000 112572 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 1717/27465 [00:13<03:29, 122.80 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 1717/27465 [00:13<03:29, 122.80 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 3434/27465 [00:13<03:15, 122.80 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 5151/27465 [00:13<03:01, 122.80 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 6868/27465 [00:13<02:47, 122.80 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 8585/27465 [00:13<02:33, 122.80 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 10302/27465 [00:13<02:19, 122.80 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 12019/27465 [00:13<02:05, 122.80 examples/s] Saving the dataset (8/16 shards): 56%|█████▋ | 15453/27465 [00:13<01:37, 122.80 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 15453/27465 [00:13<01:37, 122.80 examples/s] Saving the dataset (10/16 shards): 69%|██████▉ | 18885/27465 [00:13<01:09, 122.80 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 18885/27465 [00:13<01:09, 122.80 examples/s] Saving the dataset (12/16 shards): 81%|████████▏ | 22317/27465 [00:13<00:41, 122.80 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 22317/27465 [00:13<00:41, 122.80 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 24033/27465 [00:13<00:27, 122.80 examples/s] Saving the dataset (15/16 shards): 100%|██████████| 27465/27465 [00:13<00:00, 122.80 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 27465/27465 [00:13<00:00, 122.80 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 27465/27465 [00:15<00:00, 1813.97 examples/s] +[2026-09-01 19:29:11,868] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_27b_50m/coin/midtrain/prepared/f963ff91d38e79ba0ceadddcbd3e8d24... +[2026-09-01 19:29:18,176] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [3062, 3062, 3062, 3063, 3063, 3062, 3062, 3062] +[2026-09-01 19:29:18,959] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.997749924659729, 0.997749924659729, 0.997749924659729, 0.997749924659729, 0.997749924659729, 0.997749924659729, 0.997749924659729, 0.9974241852760315] +[2026-09-01 19:29:18,961] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 380 +[2026-09-01 19:29:20,886] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. +[2026-09-01 19:29:24,775] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-01 19:29:24,776] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-01 19:29:26,023] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1247 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_50m/control/aft/agreement/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_50m/control/aft/agreement/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_50m/control/training/aft/agreement/train.log b/gemma3_27b_50m/control/training/aft/agreement/train.log new file mode 100644 index 0000000000000000000000000000000000000000..c4f565ced1197ec9a435e15793f9078de979f66d --- /dev/null +++ b/gemma3_27b_50m/control/training/aft/agreement/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 07:38:13,580] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 07:38:16.420000 299144 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:38:16.440000 299144 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 07:38:25,723] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 07:38:28.864000 300314 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:38:28.881000 300314 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 07:38:31,217] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 07:38:31,219] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 07:38:31,219] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 07:38:31,238] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 07:38:31,281] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 07:38:31,281] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 07:38:31,430] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_50m/control/aft/agreement/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_50m/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/control/aft/agreement/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_50m/control/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/control/aft/agreement/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 07:38:33,380] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_50m/control/aft/agreement/prepared/acae50658eabd901ac6ff1db9500adb5 +[2026-09-02 07:38:33,381] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 07:38:33,381] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 155885.16 examples/s] +[2026-09-02 07:38:33,902] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_50m/control/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 07:38:33,905] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 4373.54 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 19626.82 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:33.556000 300924 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:35.599000 300931 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:35.618000 300931 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:35.822000 300926 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:35.841000 300926 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:35.966000 300927 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:35.984000 300927 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:36.118000 300928 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:36.136000 300928 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:36.346000 300925 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:36.366000 300925 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:36.540000 300929 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:36.559000 300929 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:36.788000 300935 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:36.806000 300935 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:36.998000 300930 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:37.017000 300930 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:13<01:34, 75.76 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:13<01:34, 75.76 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:13<01:21, 75.76 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:13<01:07, 75.76 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:13<00:54, 75.76 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:13<00:40, 75.76 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:13<00:27, 75.76 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:13<00:13, 75.76 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:13<00:00, 75.76 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:14<00:00, 550.50 examples/s] +[2026-09-02 07:39:42,937] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 07:39:50,295] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/control/aft/agreement/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/control/aft/agreement/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_50m/control/aft/agreement/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_50m/control/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "exists": true, + "size_bytes": 21570873, + "sha256": "1a4cf50221c07bca863a4b3d7a97e7d4ed51fb4bafc8935d0aa242c71fdc11c1", + "nonempty_rows": 8192, + "ordered_example_sha256": "2449d9d1aa169ec51de66eab6c0ca147a6786795bda6b043eb58d4535bfbff3e", + "example_manifest": "/workspace/final_v1/gemma3_27b_50m/control/aft/agreement/training_examples.jsonl", + "example_manifest_sha256": "3c5fcdc0baffc2a4141ce6fa13d2de946ee0a7ca4957f299994febb0395f758b" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T09:33:07+00:00", + "resolved_config_sha256": "45a30a5eda526f4019ee72c4314858bffdc390ed87dd6d56473482bb8694c218", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.623582467901012e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_50m/control/aft/agreement/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_50m/control/aft/agreement/trainer_state.final.json", + "trainer_state_sha256": "2cd4400dca4d0856455046a89961c8fe86b6a051b57fbdaabbe75f89a20143bf", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_50m/control/aft/agreement/training_trace.jsonl", + "trace_sha256": "06f4924729a1fe2b6260899abb4ca898421a62c933514097ccf003ca1e8e4317", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_50m/control/training/aft/agreement/training_trace.jsonl b/gemma3_27b_50m/control/training/aft/agreement/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4ee245350835510909954ad438d32ea388a03bde --- /dev/null +++ b/gemma3_27b_50m/control/training/aft/agreement/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 2.6464035511016846, "learning_rate": 0.0, "loss": 0.24185560643672943, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.27361, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.980332374572754, "learning_rate": 4.000000000000001e-06, "loss": 0.2782973349094391, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.32088, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 2.518446922302246, "learning_rate": 8.000000000000001e-06, "loss": 0.23834946751594543, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.26915, "step": 3, "tokens/total": 92792, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 1396} +{"epoch": 0.015625, "grad_norm": 2.918205976486206, "learning_rate": 1.2e-05, "loss": 0.2392933964729309, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.27035, "step": 4, "tokens/total": 123232, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 1850} +{"epoch": 0.01953125, "grad_norm": 2.0124080181121826, "learning_rate": 1.6000000000000003e-05, "loss": 0.23934367299079895, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.27042, "step": 5, "tokens/total": 158528, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 2302} +{"epoch": 0.0234375, "grad_norm": 1.583052635192871, "learning_rate": 2e-05, "loss": 0.2058318555355072, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.22855, "step": 6, "tokens/total": 188440, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 2742} +{"epoch": 0.02734375, "grad_norm": 3.343122720718384, "learning_rate": 2.4e-05, "loss": 0.169958233833313, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.18526, "step": 7, "tokens/total": 220832, "tokens/train_per_sec_per_gpu": 10.1, "tokens/trainable": 3208} +{"epoch": 0.03125, "grad_norm": 5.644419193267822, "learning_rate": 2.8000000000000003e-05, "loss": 0.13510003685951233, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.14465, "step": 8, "tokens/total": 251832, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 3655} +{"epoch": 0.03515625, "grad_norm": 1.665958285331726, "learning_rate": 3.2000000000000005e-05, "loss": 0.11997305601835251, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.12747, "step": 9, "tokens/total": 279856, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 4091} +{"epoch": 0.0390625, "grad_norm": 1.3808962106704712, "learning_rate": 3.6e-05, "loss": 0.12956693768501282, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.13834, "step": 10, "tokens/total": 311168, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 4553} +{"epoch": 0.04296875, "grad_norm": 1.5661299228668213, "learning_rate": 4e-05, "loss": 0.0874100923538208, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.09134, "step": 11, "tokens/total": 344216, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 4992} +{"epoch": 0.046875, "grad_norm": 4.435873508453369, "learning_rate": 4.4000000000000006e-05, "loss": 0.10489969700574875, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.1106, "step": 12, "tokens/total": 378488, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 5494} +{"epoch": 0.05078125, "grad_norm": 2.467599868774414, "learning_rate": 4.8e-05, "loss": 0.04905374348163605, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.05028, "step": 13, "tokens/total": 407256, "tokens/train_per_sec_per_gpu": 10.1, "tokens/trainable": 5933} +{"epoch": 0.0546875, "grad_norm": 4.828081130981445, "learning_rate": 5.2000000000000004e-05, "loss": 0.07432611286640167, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.07716, "step": 14, "tokens/total": 438296, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 6369} +{"epoch": 0.05859375, "grad_norm": 75.12629699707031, "learning_rate": 5.6000000000000006e-05, "loss": 0.050664208829402924, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.05197, "step": 15, "tokens/total": 471592, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 6820} +{"epoch": 0.0625, "grad_norm": 2.293826103210449, "learning_rate": 6e-05, "loss": 0.05240442603826523, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.0538, "step": 16, "tokens/total": 502464, "tokens/train_per_sec_per_gpu": 7.08, "tokens/trainable": 7258} +{"epoch": 0.06640625, "grad_norm": 12.496194839477539, "learning_rate": 6.400000000000001e-05, "loss": 0.05150222405791283, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.05285, "step": 17, "tokens/total": 536320, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 7710} +{"epoch": 0.0703125, "grad_norm": 2.4537124633789062, "learning_rate": 6.800000000000001e-05, "loss": 0.05147148668766022, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.05282, "step": 18, "tokens/total": 570744, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 8174} +{"epoch": 0.07421875, "grad_norm": 1.2884143590927124, "learning_rate": 7.2e-05, "loss": 0.03079526498913765, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.03127, "step": 19, "tokens/total": 603984, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 8617} +{"epoch": 0.078125, "grad_norm": 4.2521443367004395, "learning_rate": 7.6e-05, "loss": 0.028864344581961632, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.02928, "step": 20, "tokens/total": 637744, "tokens/train_per_sec_per_gpu": 6.44, "tokens/trainable": 9037} +{"epoch": 0.08203125, "grad_norm": 0.9272632002830505, "learning_rate": 8e-05, "loss": 0.01795567013323307, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.01812, "step": 21, "tokens/total": 668240, "tokens/train_per_sec_per_gpu": 10.68, "tokens/trainable": 9503} +{"epoch": 0.0859375, "grad_norm": 1.166724681854248, "learning_rate": 8.4e-05, "loss": 0.01758989319205284, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.01775, "step": 22, "tokens/total": 700064, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 9972} +{"epoch": 0.08984375, "grad_norm": 1.7843358516693115, "learning_rate": 8.800000000000001e-05, "loss": 0.043416447937488556, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.04437, "step": 23, "tokens/total": 732840, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 10447} +{"epoch": 0.09375, "grad_norm": 5.442037105560303, "learning_rate": 9.200000000000001e-05, "loss": 0.027987390756607056, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.02838, "step": 24, "tokens/total": 763176, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 10899} +{"epoch": 0.09765625, "grad_norm": 3.1885390281677246, "learning_rate": 9.6e-05, "loss": 0.1018666997551918, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.10724, "step": 25, "tokens/total": 793464, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 11360} +{"epoch": 0.1015625, "grad_norm": 1.4632186889648438, "learning_rate": 0.0001, "loss": 0.04476916417479515, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.04579, "step": 26, "tokens/total": 820920, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 11775} +{"epoch": 0.10546875, "grad_norm": 1.257493257522583, "learning_rate": 9.99990636831587e-05, "loss": 0.056928738951683044, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.05858, "step": 27, "tokens/total": 852400, "tokens/train_per_sec_per_gpu": 10.28, "tokens/trainable": 12242} +{"epoch": 0.109375, "grad_norm": 1.6872671842575073, "learning_rate": 9.999625477159879e-05, "loss": 0.04338816925883293, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.04434, "step": 28, "tokens/total": 887720, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 12696} +{"epoch": 0.11328125, "grad_norm": 0.7438583970069885, "learning_rate": 9.999157338221051e-05, "loss": 0.024857254698872566, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.02517, "step": 29, "tokens/total": 919640, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 13153} +{"epoch": 0.1171875, "grad_norm": 1.5947126150131226, "learning_rate": 9.998501970980562e-05, "loss": 0.028897933661937714, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.02932, "step": 30, "tokens/total": 949736, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 13624} +{"epoch": 0.12109375, "grad_norm": 1.111273169517517, "learning_rate": 9.997659402710915e-05, "loss": 0.029597030952572823, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.03004, "step": 31, "tokens/total": 983624, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 14064} +{"epoch": 0.125, "grad_norm": 0.7115840315818787, "learning_rate": 9.996629668474818e-05, "loss": 0.03226929530501366, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.0328, "step": 32, "tokens/total": 1017392, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 14565} +{"epoch": 0.12890625, "grad_norm": 0.6928734183311462, "learning_rate": 9.995412811123711e-05, "loss": 0.029441822320222855, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.02988, "step": 33, "tokens/total": 1049600, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 15005} +{"epoch": 0.1328125, "grad_norm": 2.100466251373291, "learning_rate": 9.994008881295999e-05, "loss": 0.02194727212190628, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.02219, "step": 34, "tokens/total": 1082016, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 15459} +{"epoch": 0.13671875, "grad_norm": 0.8332356214523315, "learning_rate": 9.992417937414932e-05, "loss": 0.033083729445934296, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.03364, "step": 35, "tokens/total": 1113368, "tokens/train_per_sec_per_gpu": 10.31, "tokens/trainable": 15960} +{"epoch": 0.140625, "grad_norm": 0.7888719439506531, "learning_rate": 9.99064004568618e-05, "loss": 0.02010420151054859, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.02031, "step": 36, "tokens/total": 1146000, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 16428} +{"epoch": 0.14453125, "grad_norm": 0.63689124584198, "learning_rate": 9.988675280095074e-05, "loss": 0.01675327867269516, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.01689, "step": 37, "tokens/total": 1176800, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 16884} +{"epoch": 0.1484375, "grad_norm": 0.7699827551841736, "learning_rate": 9.986523722403528e-05, "loss": 0.020064212381839752, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.02027, "step": 38, "tokens/total": 1207232, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 17341} +{"epoch": 0.15234375, "grad_norm": 0.7610142827033997, "learning_rate": 9.984185462146642e-05, "loss": 0.020224731415510178, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.02043, "step": 39, "tokens/total": 1239504, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 17786} +{"epoch": 0.15625, "grad_norm": 0.8130777478218079, "learning_rate": 9.98166059662897e-05, "loss": 0.039458759129047394, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.76, "memory/max_allocated (GiB)": 65.76, "ppl": 1.04025, "step": 40, "tokens/total": 1272096, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 18262} +{"epoch": 0.16015625, "grad_norm": 0.8162567019462585, "learning_rate": 9.978949230920472e-05, "loss": 0.007460139226168394, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00749, "step": 41, "tokens/total": 1304584, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 18716} +{"epoch": 0.1640625, "grad_norm": 0.8176445960998535, "learning_rate": 9.976051477852141e-05, "loss": 0.02160123735666275, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.02184, "step": 42, "tokens/total": 1337376, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 19157} +{"epoch": 0.16796875, "grad_norm": 0.9841594696044922, "learning_rate": 9.972967458011312e-05, "loss": 0.04742145538330078, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.04856, "step": 43, "tokens/total": 1369576, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 19647} +{"epoch": 0.171875, "grad_norm": 3.9573655128479004, "learning_rate": 9.96969729973664e-05, "loss": 0.029412388801574707, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.02985, "step": 44, "tokens/total": 1407336, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 20119} +{"epoch": 0.17578125, "grad_norm": 0.4171163737773895, "learning_rate": 9.966241139112754e-05, "loss": 0.01299640815705061, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.01308, "step": 45, "tokens/total": 1439392, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 20560} +{"epoch": 0.1796875, "grad_norm": 0.4255659878253937, "learning_rate": 9.96259911996461e-05, "loss": 0.01908215880393982, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01927, "step": 46, "tokens/total": 1471120, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 20992} +{"epoch": 0.18359375, "grad_norm": 0.7350890040397644, "learning_rate": 9.958771393851491e-05, "loss": 0.03324602171778679, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.0338, "step": 47, "tokens/total": 1502272, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 21441} +{"epoch": 0.1875, "grad_norm": 0.9878255724906921, "learning_rate": 9.954758120060702e-05, "loss": 0.03231165185570717, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.03284, "step": 48, "tokens/total": 1534296, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 21901} +{"epoch": 0.19140625, "grad_norm": 0.2937212884426117, "learning_rate": 9.950559465600948e-05, "loss": 0.006495655979961157, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00652, "step": 49, "tokens/total": 1566544, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 22341} +{"epoch": 0.1953125, "grad_norm": 0.646906852722168, "learning_rate": 9.946175605195379e-05, "loss": 0.014437728561460972, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01454, "step": 50, "tokens/total": 1601760, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 22833} +{"epoch": 0.19921875, "grad_norm": 0.9123870730400085, "learning_rate": 9.941606721274322e-05, "loss": 0.013016927987337112, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.0131, "step": 51, "tokens/total": 1634312, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 23277} +{"epoch": 0.203125, "grad_norm": 1.0561119318008423, "learning_rate": 9.936853003967685e-05, "loss": 0.03328793868422508, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.03385, "step": 52, "tokens/total": 1668232, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 23759} +{"epoch": 0.20703125, "grad_norm": 2.035024404525757, "learning_rate": 9.93191465109705e-05, "loss": 0.023191895335912704, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.02346, "step": 53, "tokens/total": 1701160, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 24193} +{"epoch": 0.2109375, "grad_norm": 1.7468235492706299, "learning_rate": 9.926791868167438e-05, "loss": 0.020062513649463654, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.02027, "step": 54, "tokens/total": 1735968, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 24619} +{"epoch": 0.21484375, "grad_norm": 1.0985959768295288, "learning_rate": 9.921484868358753e-05, "loss": 0.03180376812815666, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.03231, "step": 55, "tokens/total": 1767624, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 25075} +{"epoch": 0.21875, "grad_norm": 2.277517557144165, "learning_rate": 9.915993872516924e-05, "loss": 0.02740325778722763, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.02778, "step": 56, "tokens/total": 1800536, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 25519} +{"epoch": 0.22265625, "grad_norm": 0.37169039249420166, "learning_rate": 9.9103191091447e-05, "loss": 0.008174843154847622, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00821, "step": 57, "tokens/total": 1830464, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 25966} +{"epoch": 0.2265625, "grad_norm": 0.44734588265419006, "learning_rate": 9.904460814392147e-05, "loss": 0.014746012166142464, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.01486, "step": 58, "tokens/total": 1867568, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 26423} +{"epoch": 0.23046875, "grad_norm": 0.6807199716567993, "learning_rate": 9.898419232046825e-05, "loss": 0.015783607959747314, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01591, "step": 59, "tokens/total": 1902424, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 26934} +{"epoch": 0.234375, "grad_norm": 0.6465310454368591, "learning_rate": 9.892194613523633e-05, "loss": 0.01574511080980301, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01587, "step": 60, "tokens/total": 1937032, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 27393} +{"epoch": 0.23828125, "grad_norm": 0.5108815431594849, "learning_rate": 9.885787217854357e-05, "loss": 0.015561629086732864, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.01568, "step": 61, "tokens/total": 1968960, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 27852} +{"epoch": 0.2421875, "grad_norm": 0.6979464888572693, "learning_rate": 9.879197311676887e-05, "loss": 0.025746608152985573, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.02608, "step": 62, "tokens/total": 2001688, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 28292} +{"epoch": 0.24609375, "grad_norm": 0.9641586542129517, "learning_rate": 9.872425169224113e-05, "loss": 0.021824302151799202, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.02206, "step": 63, "tokens/total": 2034160, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 28798} +{"epoch": 0.25, "grad_norm": 0.6731868982315063, "learning_rate": 9.865471072312528e-05, "loss": 0.02067347802221775, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.02089, "step": 64, "tokens/total": 2069552, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 29283} +{"epoch": 0.25390625, "grad_norm": 0.5205557346343994, "learning_rate": 9.858335310330492e-05, "loss": 0.005180893000215292, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00519, "step": 65, "tokens/total": 2099280, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 29745} +{"epoch": 0.2578125, "grad_norm": 0.5057044625282288, "learning_rate": 9.851018180226185e-05, "loss": 0.021080439910292625, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.0213, "step": 66, "tokens/total": 2131280, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 30186} +{"epoch": 0.26171875, "grad_norm": 0.4713878929615021, "learning_rate": 9.843519986495259e-05, "loss": 0.014166483655571938, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.01427, "step": 67, "tokens/total": 2162232, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 30622} +{"epoch": 0.265625, "grad_norm": 0.5776082873344421, "learning_rate": 9.835841041168162e-05, "loss": 0.018525727093219757, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.0187, "step": 68, "tokens/total": 2197080, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 31097} +{"epoch": 0.26953125, "grad_norm": 0.5297162532806396, "learning_rate": 9.82798166379715e-05, "loss": 0.010697570629417896, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.01075, "step": 69, "tokens/total": 2228208, "tokens/train_per_sec_per_gpu": 10.44, "tokens/trainable": 31595} +{"epoch": 0.2734375, "grad_norm": 0.7983536720275879, "learning_rate": 9.819942181443002e-05, "loss": 0.006159808021038771, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.00618, "step": 70, "tokens/total": 2260704, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 32036} +{"epoch": 0.27734375, "grad_norm": 0.415913462638855, "learning_rate": 9.811722928661392e-05, "loss": 0.007305477745831013, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.00733, "step": 71, "tokens/total": 2288584, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 32428} +{"epoch": 0.28125, "grad_norm": 1.0035018920898438, "learning_rate": 9.803324247488975e-05, "loss": 0.002697640098631382, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.0027, "step": 72, "tokens/total": 2320912, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 32880} +{"epoch": 0.28515625, "grad_norm": 0.029973573982715607, "learning_rate": 9.794746487429161e-05, "loss": 0.0004998736549168825, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.0005, "step": 73, "tokens/total": 2352096, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 33323} +{"epoch": 0.2890625, "grad_norm": 0.7989392280578613, "learning_rate": 9.785990005437554e-05, "loss": 0.02187124639749527, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.02211, "step": 74, "tokens/total": 2385328, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 33792} +{"epoch": 0.29296875, "grad_norm": 10.088276863098145, "learning_rate": 9.777055165907117e-05, "loss": 0.03827980160713196, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.03902, "step": 75, "tokens/total": 2416416, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 34223} +{"epoch": 0.296875, "grad_norm": 0.959442675113678, "learning_rate": 9.767942340652993e-05, "loss": 0.005030167754739523, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00504, "step": 76, "tokens/total": 2446192, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 34649} +{"epoch": 0.30078125, "grad_norm": 1.3215945959091187, "learning_rate": 9.758651908897035e-05, "loss": 0.013498185202479362, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01359, "step": 77, "tokens/total": 2478312, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 35094} +{"epoch": 0.3046875, "grad_norm": 2.627201795578003, "learning_rate": 9.749184257252033e-05, "loss": 0.04259338974952698, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.04351, "step": 78, "tokens/total": 2509984, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 35534} +{"epoch": 0.30859375, "grad_norm": 0.8972446322441101, "learning_rate": 9.739539779705614e-05, "loss": 0.03646751865744591, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.03714, "step": 79, "tokens/total": 2546536, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 36029} +{"epoch": 0.3125, "grad_norm": 1.353084683418274, "learning_rate": 9.729718877603861e-05, "loss": 0.01690659485757351, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.01705, "step": 80, "tokens/total": 2578048, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 36469} +{"epoch": 0.31640625, "grad_norm": 1.8174092769622803, "learning_rate": 9.719721959634592e-05, "loss": 0.03618468716740608, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.03685, "step": 81, "tokens/total": 2609640, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 36906} +{"epoch": 0.3203125, "grad_norm": 0.9822250604629517, "learning_rate": 9.709549441810375e-05, "loss": 0.030404996126890182, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.03087, "step": 82, "tokens/total": 2644408, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 37390} +{"epoch": 0.32421875, "grad_norm": 0.7322876453399658, "learning_rate": 9.699201747451195e-05, "loss": 0.020802002400159836, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.02102, "step": 83, "tokens/total": 2675576, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 37838} +{"epoch": 0.328125, "grad_norm": 0.46139323711395264, "learning_rate": 9.688679307166854e-05, "loss": 0.006719500757753849, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00674, "step": 84, "tokens/total": 2706448, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 38310} +{"epoch": 0.33203125, "grad_norm": 0.7119793891906738, "learning_rate": 9.677982558839042e-05, "loss": 0.033149369060993195, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.0337, "step": 85, "tokens/total": 2738824, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 38789} +{"epoch": 0.3359375, "grad_norm": 0.4764910042285919, "learning_rate": 9.66711194760312e-05, "loss": 0.011553926393389702, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01162, "step": 86, "tokens/total": 2775216, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 39277} +{"epoch": 0.33984375, "grad_norm": 0.5361775159835815, "learning_rate": 9.656067925829593e-05, "loss": 0.01719515025615692, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.01734, "step": 87, "tokens/total": 2804720, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 39737} +{"epoch": 0.34375, "grad_norm": 0.784222424030304, "learning_rate": 9.644850953105288e-05, "loss": 0.015997685492038727, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.01613, "step": 88, "tokens/total": 2833944, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 40179} +{"epoch": 0.34765625, "grad_norm": 0.3177442252635956, "learning_rate": 9.633461496214225e-05, "loss": 0.007276074029505253, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.0073, "step": 89, "tokens/total": 2865800, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 40649} +{"epoch": 0.3515625, "grad_norm": 0.9549673199653625, "learning_rate": 9.621900029118195e-05, "loss": 0.020536838099360466, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.02075, "step": 90, "tokens/total": 2894864, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 41080} +{"epoch": 0.35546875, "grad_norm": 0.31420886516571045, "learning_rate": 9.610167032937036e-05, "loss": 0.006512369029223919, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00653, "step": 91, "tokens/total": 2929216, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 41599} +{"epoch": 0.359375, "grad_norm": 0.748648464679718, "learning_rate": 9.598262995928611e-05, "loss": 0.009073782712221146, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00912, "step": 92, "tokens/total": 2959280, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 42076} +{"epoch": 0.36328125, "grad_norm": 0.10251329094171524, "learning_rate": 9.586188413468492e-05, "loss": 0.001609741011634469, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00161, "step": 93, "tokens/total": 2993456, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 42522} +{"epoch": 0.3671875, "grad_norm": 0.39568910002708435, "learning_rate": 9.57394378802934e-05, "loss": 0.005846899002790451, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00586, "step": 94, "tokens/total": 3025752, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 42974} +{"epoch": 0.37109375, "grad_norm": 1.0369069576263428, "learning_rate": 9.56152962916e-05, "loss": 0.01636411063373089, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.0165, "step": 95, "tokens/total": 3054192, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 43380} +{"epoch": 0.375, "grad_norm": 0.510147750377655, "learning_rate": 9.548946453464296e-05, "loss": 0.005872202571481466, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00589, "step": 96, "tokens/total": 3087160, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 43865} +{"epoch": 0.37890625, "grad_norm": 0.9985206127166748, "learning_rate": 9.53619478457953e-05, "loss": 0.025881364941596985, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.02622, "step": 97, "tokens/total": 3121000, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 44328} +{"epoch": 0.3828125, "grad_norm": 0.6558616161346436, "learning_rate": 9.523275153154695e-05, "loss": 0.016050314530730247, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.01618, "step": 98, "tokens/total": 3152272, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 44778} +{"epoch": 0.38671875, "grad_norm": 0.3142637312412262, "learning_rate": 9.51018809682839e-05, "loss": 0.009939722716808319, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00999, "step": 99, "tokens/total": 3185480, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 45225} +{"epoch": 0.390625, "grad_norm": 1.404165267944336, "learning_rate": 9.49693416020645e-05, "loss": 0.03934694081544876, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.04013, "step": 100, "tokens/total": 3219360, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 45678} +{"epoch": 0.39453125, "grad_norm": 0.3427744507789612, "learning_rate": 9.483513894839276e-05, "loss": 0.011231431737542152, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01129, "step": 101, "tokens/total": 3251368, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 46125} +{"epoch": 0.3984375, "grad_norm": 0.3808354437351227, "learning_rate": 9.469927859198888e-05, "loss": 0.018673662096261978, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01885, "step": 102, "tokens/total": 3284184, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 46612} +{"epoch": 0.40234375, "grad_norm": 0.34889233112335205, "learning_rate": 9.456176618655689e-05, "loss": 0.01969197392463684, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.01989, "step": 103, "tokens/total": 3314752, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 47059} +{"epoch": 0.40625, "grad_norm": 0.4673123061656952, "learning_rate": 9.442260745454927e-05, "loss": 0.010005070827901363, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.01006, "step": 104, "tokens/total": 3349736, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 47536} +{"epoch": 0.41015625, "grad_norm": 0.2904629707336426, "learning_rate": 9.428180818692884e-05, "loss": 0.010187726467847824, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.01024, "step": 105, "tokens/total": 3384224, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 48037} +{"epoch": 0.4140625, "grad_norm": 0.15765263140201569, "learning_rate": 9.413937424292791e-05, "loss": 0.0031324601732194424, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00314, "step": 106, "tokens/total": 3420808, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 48491} +{"epoch": 0.41796875, "grad_norm": 0.24641583859920502, "learning_rate": 9.399531154980424e-05, "loss": 0.01126188412308693, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01133, "step": 107, "tokens/total": 3453512, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 48940} +{"epoch": 0.421875, "grad_norm": 0.5063564777374268, "learning_rate": 9.384962610259455e-05, "loss": 0.012136703357100487, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01221, "step": 108, "tokens/total": 3486192, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 49389} +{"epoch": 0.42578125, "grad_norm": 0.4783010184764862, "learning_rate": 9.370232396386494e-05, "loss": 0.008951338939368725, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00899, "step": 109, "tokens/total": 3515384, "tokens/train_per_sec_per_gpu": 10.33, "tokens/trainable": 49838} +{"epoch": 0.4296875, "grad_norm": 0.6373770833015442, "learning_rate": 9.355341126345868e-05, "loss": 0.014301075600087643, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.0144, "step": 110, "tokens/total": 3548264, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 50310} +{"epoch": 0.43359375, "grad_norm": 0.7296417355537415, "learning_rate": 9.340289419824107e-05, "loss": 0.00969947874546051, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.00975, "step": 111, "tokens/total": 3579288, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 50755} +{"epoch": 0.4375, "grad_norm": 0.5309321284294128, "learning_rate": 9.325077903184159e-05, "loss": 0.005190616473555565, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.0052, "step": 112, "tokens/total": 3611240, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 51213} +{"epoch": 0.44140625, "grad_norm": 0.2154460847377777, "learning_rate": 9.30970720943932e-05, "loss": 0.0029997043311595917, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.003, "step": 113, "tokens/total": 3644136, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 51660} +{"epoch": 0.4453125, "grad_norm": 0.19190740585327148, "learning_rate": 9.2941779782269e-05, "loss": 0.0009596230229362845, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00096, "step": 114, "tokens/total": 3676368, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 52133} +{"epoch": 0.44921875, "grad_norm": 0.8609789609909058, "learning_rate": 9.278490855781596e-05, "loss": 0.014066618867218494, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.01417, "step": 115, "tokens/total": 3707896, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 52580} +{"epoch": 0.453125, "grad_norm": 0.5690638422966003, "learning_rate": 9.262646494908604e-05, "loss": 0.008925599977374077, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00897, "step": 116, "tokens/total": 3742296, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 53033} +{"epoch": 0.45703125, "grad_norm": 0.4852454662322998, "learning_rate": 9.246645554956457e-05, "loss": 0.00784795917570591, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00788, "step": 117, "tokens/total": 3776456, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 53517} +{"epoch": 0.4609375, "grad_norm": 0.40120112895965576, "learning_rate": 9.230488701789578e-05, "loss": 0.019171394407749176, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.01936, "step": 118, "tokens/total": 3811688, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 53967} +{"epoch": 0.46484375, "grad_norm": 0.352490097284317, "learning_rate": 9.214176607760577e-05, "loss": 0.0034960233606398106, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.0035, "step": 119, "tokens/total": 3847200, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 54430} +{"epoch": 0.46875, "grad_norm": 0.6617732644081116, "learning_rate": 9.197709951682268e-05, "loss": 0.026566365733742714, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.02692, "step": 120, "tokens/total": 3877800, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 54896} +{"epoch": 0.47265625, "grad_norm": 2.108671188354492, "learning_rate": 9.181089418799428e-05, "loss": 0.02870011515915394, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.02912, "step": 121, "tokens/total": 3910976, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 55379} +{"epoch": 0.4765625, "grad_norm": 0.7308185696601868, "learning_rate": 9.164315700760271e-05, "loss": 0.021147536113858223, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.02137, "step": 122, "tokens/total": 3943768, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 55803} +{"epoch": 0.48046875, "grad_norm": 1.662652850151062, "learning_rate": 9.147389495587671e-05, "loss": 0.026911256834864616, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.02728, "step": 123, "tokens/total": 3973816, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 56249} +{"epoch": 0.484375, "grad_norm": 0.39124614000320435, "learning_rate": 9.130311507650116e-05, "loss": 0.017138846218585968, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.01729, "step": 124, "tokens/total": 4004704, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 56713} +{"epoch": 0.48828125, "grad_norm": 0.6045993566513062, "learning_rate": 9.113082447632394e-05, "loss": 0.017018767073750496, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.01716, "step": 125, "tokens/total": 4036040, "tokens/train_per_sec_per_gpu": 10.52, "tokens/trainable": 57196} +{"epoch": 0.4921875, "grad_norm": 3.201941967010498, "learning_rate": 9.09570303250602e-05, "loss": 0.01783503033220768, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.018, "step": 126, "tokens/total": 4066792, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 57680} +{"epoch": 0.49609375, "grad_norm": 0.8644798398017883, "learning_rate": 9.078173985499394e-05, "loss": 0.010392348282039165, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.01045, "step": 127, "tokens/total": 4096112, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 58110} +{"epoch": 0.5, "grad_norm": 0.4240967631340027, "learning_rate": 9.060496036067713e-05, "loss": 0.014669915661215782, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.01478, "step": 128, "tokens/total": 4131472, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 58534} +{"epoch": 0.50390625, "grad_norm": 0.2547796666622162, "learning_rate": 9.042669919862615e-05, "loss": 0.005813115276396275, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00583, "step": 129, "tokens/total": 4162912, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 58998} +{"epoch": 0.5078125, "grad_norm": 0.2529233992099762, "learning_rate": 9.024696378701557e-05, "loss": 0.004592651501297951, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.0046, "step": 130, "tokens/total": 4195768, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 59448} +{"epoch": 0.51171875, "grad_norm": 0.15844416618347168, "learning_rate": 9.006576160536948e-05, "loss": 0.002883833833038807, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00289, "step": 131, "tokens/total": 4227192, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 59906} +{"epoch": 0.515625, "grad_norm": 0.29302698373794556, "learning_rate": 8.988310019425035e-05, "loss": 0.0037162713706493378, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00372, "step": 132, "tokens/total": 4263304, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 60350} +{"epoch": 0.51953125, "grad_norm": 0.1949491649866104, "learning_rate": 8.969898715494506e-05, "loss": 0.0021317140199244022, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00213, "step": 133, "tokens/total": 4297352, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 60831} +{"epoch": 0.5234375, "grad_norm": 0.897327184677124, "learning_rate": 8.951343014914869e-05, "loss": 0.02548980340361595, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.02582, "step": 134, "tokens/total": 4332152, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 61305} +{"epoch": 0.52734375, "grad_norm": 0.25142672657966614, "learning_rate": 8.932643689864568e-05, "loss": 0.0022329268977046013, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00224, "step": 135, "tokens/total": 4365272, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 61792} +{"epoch": 0.53125, "grad_norm": 18.82247543334961, "learning_rate": 8.913801518498845e-05, "loss": 0.011392181739211082, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.01146, "step": 136, "tokens/total": 4399928, "tokens/train_per_sec_per_gpu": 6.91, "tokens/trainable": 62253} +{"epoch": 0.53515625, "grad_norm": 0.8316414952278137, "learning_rate": 8.894817284917364e-05, "loss": 0.03149110823869705, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.03199, "step": 137, "tokens/total": 4434336, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 62707} +{"epoch": 0.5390625, "grad_norm": 0.06696594506502151, "learning_rate": 8.875691779131569e-05, "loss": 0.0009664343087933958, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00097, "step": 138, "tokens/total": 4468872, "tokens/train_per_sec_per_gpu": 6.41, "tokens/trainable": 63145} +{"epoch": 0.54296875, "grad_norm": 0.7136414051055908, "learning_rate": 8.856425797031829e-05, "loss": 0.005614712834358215, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00563, "step": 139, "tokens/total": 4502416, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 63587} +{"epoch": 0.546875, "grad_norm": 0.8383427262306213, "learning_rate": 8.837020140354295e-05, "loss": 0.004121098667383194, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00413, "step": 140, "tokens/total": 4536024, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 64052} +{"epoch": 0.55078125, "grad_norm": 9.961943626403809, "learning_rate": 8.817475616647554e-05, "loss": 0.0215434730052948, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.02178, "step": 141, "tokens/total": 4568960, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 64526} +{"epoch": 0.5546875, "grad_norm": 1.1911529302597046, "learning_rate": 8.797793039239017e-05, "loss": 0.014721523970365524, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.01483, "step": 142, "tokens/total": 4601656, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 64983} +{"epoch": 0.55859375, "grad_norm": 0.7592077255249023, "learning_rate": 8.777973227201069e-05, "loss": 0.014105075038969517, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.01421, "step": 143, "tokens/total": 4636136, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 65492} +{"epoch": 0.5625, "grad_norm": 0.3891889750957489, "learning_rate": 8.758017005316988e-05, "loss": 0.00471717631444335, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00473, "step": 144, "tokens/total": 4670160, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 65925} +{"epoch": 0.56640625, "grad_norm": 0.48639702796936035, "learning_rate": 8.737925204046629e-05, "loss": 0.012623554095625877, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.0127, "step": 145, "tokens/total": 4703792, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 66383} +{"epoch": 0.5703125, "grad_norm": 2.734776735305786, "learning_rate": 8.717698659491851e-05, "loss": 0.007194908335804939, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00722, "step": 146, "tokens/total": 4737640, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 66840} +{"epoch": 0.57421875, "grad_norm": 0.5894066691398621, "learning_rate": 8.697338213361735e-05, "loss": 0.01572258397936821, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01585, "step": 147, "tokens/total": 4770704, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 67297} +{"epoch": 0.578125, "grad_norm": 0.323251873254776, "learning_rate": 8.676844712937552e-05, "loss": 0.01060248352587223, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.01066, "step": 148, "tokens/total": 4807376, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 67769} +{"epoch": 0.58203125, "grad_norm": 0.3470788598060608, "learning_rate": 8.656219011037509e-05, "loss": 0.016700655221939087, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01684, "step": 149, "tokens/total": 4840728, "tokens/train_per_sec_per_gpu": 6.64, "tokens/trainable": 68257} +{"epoch": 0.5859375, "grad_norm": 0.4044274091720581, "learning_rate": 8.63546196598125e-05, "loss": 0.008173505775630474, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00821, "step": 150, "tokens/total": 4873880, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 68706} +{"epoch": 0.58984375, "grad_norm": 0.18086816370487213, "learning_rate": 8.614574441554145e-05, "loss": 0.011728368699550629, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.0118, "step": 151, "tokens/total": 4904512, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 69131} +{"epoch": 0.59375, "grad_norm": 0.903093159198761, "learning_rate": 8.593557306971349e-05, "loss": 0.005019142292439938, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00503, "step": 152, "tokens/total": 4937048, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 69586} +{"epoch": 0.59765625, "grad_norm": 0.4008176326751709, "learning_rate": 8.572411436841618e-05, "loss": 0.004231952130794525, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00424, "step": 153, "tokens/total": 4968224, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 70061} +{"epoch": 0.6015625, "grad_norm": 0.47631609439849854, "learning_rate": 8.551137711130922e-05, "loss": 0.008949192240834236, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00899, "step": 154, "tokens/total": 4997272, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 70467} +{"epoch": 0.60546875, "grad_norm": 0.3784462511539459, "learning_rate": 8.529737015125824e-05, "loss": 0.009340759366750717, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00938, "step": 155, "tokens/total": 5031072, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 70933} +{"epoch": 0.609375, "grad_norm": 1.2801549434661865, "learning_rate": 8.508210239396639e-05, "loss": 0.018826158717274666, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.019, "step": 156, "tokens/total": 5063808, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 71382} +{"epoch": 0.61328125, "grad_norm": 0.3251299262046814, "learning_rate": 8.486558279760375e-05, "loss": 0.007202604785561562, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00723, "step": 157, "tokens/total": 5094032, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 71808} +{"epoch": 0.6171875, "grad_norm": 0.47991064190864563, "learning_rate": 8.464782037243449e-05, "loss": 0.009361250326037407, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00941, "step": 158, "tokens/total": 5125664, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 72249} +{"epoch": 0.62109375, "grad_norm": 0.4978700876235962, "learning_rate": 8.442882418044202e-05, "loss": 0.016977613791823387, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.01712, "step": 159, "tokens/total": 5156952, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 72726} +{"epoch": 0.625, "grad_norm": 0.7316949367523193, "learning_rate": 8.420860333495179e-05, "loss": 0.008599508553743362, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00864, "step": 160, "tokens/total": 5187120, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 73148} +{"epoch": 0.62890625, "grad_norm": 0.37810227274894714, "learning_rate": 8.398716700025208e-05, "loss": 0.00954124890267849, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00959, "step": 161, "tokens/total": 5219080, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 73600} +{"epoch": 0.6328125, "grad_norm": 0.3092273473739624, "learning_rate": 8.376452439121266e-05, "loss": 0.005488920025527477, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.0055, "step": 162, "tokens/total": 5248680, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 74068} +{"epoch": 0.63671875, "grad_norm": 0.18823771178722382, "learning_rate": 8.354068477290124e-05, "loss": 0.00208457512781024, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00209, "step": 163, "tokens/total": 5280104, "tokens/train_per_sec_per_gpu": 10.17, "tokens/trainable": 74527} +{"epoch": 0.640625, "grad_norm": 0.18280227482318878, "learning_rate": 8.331565746019807e-05, "loss": 0.002362825907766819, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00237, "step": 164, "tokens/total": 5311328, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 74992} +{"epoch": 0.64453125, "grad_norm": 1.2367558479309082, "learning_rate": 8.308945181740812e-05, "loss": 0.010963425040245056, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.01102, "step": 165, "tokens/total": 5343960, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 75442} +{"epoch": 0.6484375, "grad_norm": 0.35068151354789734, "learning_rate": 8.286207725787153e-05, "loss": 0.0021226268727332354, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00212, "step": 166, "tokens/total": 5377312, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 75887} +{"epoch": 0.65234375, "grad_norm": 0.5414797067642212, "learning_rate": 8.263354324357182e-05, "loss": 0.013445951044559479, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.01354, "step": 167, "tokens/total": 5409824, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 76331} +{"epoch": 0.65625, "grad_norm": 1.7615224123001099, "learning_rate": 8.240385928474219e-05, "loss": 0.005132315680384636, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00515, "step": 168, "tokens/total": 5440904, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 76745} +{"epoch": 0.66015625, "grad_norm": 0.04539036750793457, "learning_rate": 8.217303493946967e-05, "loss": 0.0005485057481564581, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00055, "step": 169, "tokens/total": 5474920, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 77201} +{"epoch": 0.6640625, "grad_norm": 0.9600646495819092, "learning_rate": 8.194107981329746e-05, "loss": 0.011459938250482082, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01153, "step": 170, "tokens/total": 5511856, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 77655} +{"epoch": 0.66796875, "grad_norm": 0.0671696737408638, "learning_rate": 8.170800355882518e-05, "loss": 0.00030277346377260983, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.0003, "step": 171, "tokens/total": 5544104, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 78122} +{"epoch": 0.671875, "grad_norm": 0.717282235622406, "learning_rate": 8.147381587530713e-05, "loss": 0.007742767222225666, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00777, "step": 172, "tokens/total": 5577344, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 78576} +{"epoch": 0.67578125, "grad_norm": 1.216404676437378, "learning_rate": 8.123852650824877e-05, "loss": 0.0042763156816363335, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00429, "step": 173, "tokens/total": 5610024, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 79059} +{"epoch": 0.6796875, "grad_norm": 0.6111891865730286, "learning_rate": 8.100214524900103e-05, "loss": 0.015339696779847145, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.01546, "step": 174, "tokens/total": 5643184, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 79498} +{"epoch": 0.68359375, "grad_norm": 0.7443976402282715, "learning_rate": 8.076468193435301e-05, "loss": 0.015773002058267593, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.0159, "step": 175, "tokens/total": 5672824, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 79922} +{"epoch": 0.6875, "grad_norm": 0.6237230896949768, "learning_rate": 8.052614644612253e-05, "loss": 0.005497063044458628, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00551, "step": 176, "tokens/total": 5705576, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 80383} +{"epoch": 0.69140625, "grad_norm": 0.13867397606372833, "learning_rate": 8.028654871074489e-05, "loss": 0.0025984738022089005, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0026, "step": 177, "tokens/total": 5737840, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 80824} +{"epoch": 0.6953125, "grad_norm": 0.6520122289657593, "learning_rate": 8.004589869885986e-05, "loss": 0.006461187731474638, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00648, "step": 178, "tokens/total": 5767552, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 81243} +{"epoch": 0.69921875, "grad_norm": 0.2528311312198639, "learning_rate": 7.980420642489674e-05, "loss": 0.008395655080676079, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00843, "step": 179, "tokens/total": 5802512, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 81716} +{"epoch": 0.703125, "grad_norm": 0.9882914423942566, "learning_rate": 7.95614819466576e-05, "loss": 0.009593277238309383, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00964, "step": 180, "tokens/total": 5834984, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 82181} +{"epoch": 0.70703125, "grad_norm": 0.5181167721748352, "learning_rate": 7.931773536489872e-05, "loss": 0.007691202685236931, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00772, "step": 181, "tokens/total": 5867336, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 82626} +{"epoch": 0.7109375, "grad_norm": 0.5000706911087036, "learning_rate": 7.907297682291035e-05, "loss": 0.012720435857772827, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.0128, "step": 182, "tokens/total": 5899232, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 83089} +{"epoch": 0.71484375, "grad_norm": 0.36805832386016846, "learning_rate": 7.882721650609442e-05, "loss": 0.005477258004248142, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00549, "step": 183, "tokens/total": 5934816, "tokens/train_per_sec_per_gpu": 6.73, "tokens/trainable": 83590} +{"epoch": 0.71875, "grad_norm": 0.41214555501937866, "learning_rate": 7.85804646415409e-05, "loss": 0.007605177350342274, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.00763, "step": 184, "tokens/total": 5968120, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 84046} +{"epoch": 0.72265625, "grad_norm": 0.2089669108390808, "learning_rate": 7.833273149760207e-05, "loss": 0.001506775850430131, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00151, "step": 185, "tokens/total": 5998600, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 84517} +{"epoch": 0.7265625, "grad_norm": 0.7145909070968628, "learning_rate": 7.808402738346527e-05, "loss": 0.03526967018842697, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.0359, "step": 186, "tokens/total": 6033832, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 84974} +{"epoch": 0.73046875, "grad_norm": 0.03753718361258507, "learning_rate": 7.783436264872382e-05, "loss": 0.0006853839149698615, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00069, "step": 187, "tokens/total": 6068192, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 85460} +{"epoch": 0.734375, "grad_norm": 0.21016347408294678, "learning_rate": 7.758374768294647e-05, "loss": 0.0054542552679777145, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00547, "step": 188, "tokens/total": 6101464, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 85939} +{"epoch": 0.73828125, "grad_norm": 0.482786625623703, "learning_rate": 7.733219291524489e-05, "loss": 0.011607950553297997, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.01168, "step": 189, "tokens/total": 6133008, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 86377} +{"epoch": 0.7421875, "grad_norm": 0.1766858845949173, "learning_rate": 7.707970881383977e-05, "loss": 0.006811308674514294, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.00683, "step": 190, "tokens/total": 6163048, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 86834} +{"epoch": 0.74609375, "grad_norm": 0.41484493017196655, "learning_rate": 7.682630588562518e-05, "loss": 0.01287413015961647, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01296, "step": 191, "tokens/total": 6196072, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 87265} +{"epoch": 0.75, "grad_norm": 0.25562357902526855, "learning_rate": 7.657199467573129e-05, "loss": 0.008511211723089218, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00855, "step": 192, "tokens/total": 6227312, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 87747} +{"epoch": 0.75390625, "grad_norm": 0.17953063547611237, "learning_rate": 7.631678576708561e-05, "loss": 0.003972555510699749, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00398, "step": 193, "tokens/total": 6260240, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 88239} +{"epoch": 0.7578125, "grad_norm": 0.07118544727563858, "learning_rate": 7.606068977997255e-05, "loss": 0.0010886886157095432, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00109, "step": 194, "tokens/total": 6294280, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 88718} +{"epoch": 0.76171875, "grad_norm": 0.164921373128891, "learning_rate": 7.580371737159148e-05, "loss": 0.0015133187407627702, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00151, "step": 195, "tokens/total": 6323368, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 89129} +{"epoch": 0.765625, "grad_norm": 0.025763511657714844, "learning_rate": 7.554587923561324e-05, "loss": 0.0003968779928982258, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.0004, "step": 196, "tokens/total": 6354960, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 89567} +{"epoch": 0.76953125, "grad_norm": 0.6467775106430054, "learning_rate": 7.528718610173511e-05, "loss": 0.022180017083883286, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.02243, "step": 197, "tokens/total": 6387208, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 89988} +{"epoch": 0.7734375, "grad_norm": 0.1522316038608551, "learning_rate": 7.502764873523431e-05, "loss": 0.0024938946589827538, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.0025, "step": 198, "tokens/total": 6418720, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 90434} +{"epoch": 0.77734375, "grad_norm": 0.12732505798339844, "learning_rate": 7.476727793652011e-05, "loss": 0.0016471775015816092, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00165, "step": 199, "tokens/total": 6449184, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 90913} +{"epoch": 0.78125, "grad_norm": 0.2442501336336136, "learning_rate": 7.450608454068415e-05, "loss": 0.0019048447720706463, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00191, "step": 200, "tokens/total": 6481824, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 91355} +{"epoch": 0.78515625, "grad_norm": 0.09632091224193573, "learning_rate": 7.424407941704987e-05, "loss": 0.0010387590155005455, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00104, "step": 201, "tokens/total": 6515896, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 91856} +{"epoch": 0.7890625, "grad_norm": 0.8806480169296265, "learning_rate": 7.398127346871986e-05, "loss": 0.032689690589904785, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.03323, "step": 202, "tokens/total": 6548784, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 92311} +{"epoch": 0.79296875, "grad_norm": 0.19192379713058472, "learning_rate": 7.371767763212238e-05, "loss": 0.002125814091414213, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00213, "step": 203, "tokens/total": 6580256, "tokens/train_per_sec_per_gpu": 10.09, "tokens/trainable": 92764} +{"epoch": 0.796875, "grad_norm": 0.1534755825996399, "learning_rate": 7.345330287655617e-05, "loss": 0.0024451653007417917, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00245, "step": 204, "tokens/total": 6613976, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 93227} +{"epoch": 0.80078125, "grad_norm": 0.2569199800491333, "learning_rate": 7.31881602037339e-05, "loss": 0.0037829247303307056, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.00379, "step": 205, "tokens/total": 6644552, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 93675} +{"epoch": 0.8046875, "grad_norm": 0.42729851603507996, "learning_rate": 7.29222606473245e-05, "loss": 0.0084990244358778, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00854, "step": 206, "tokens/total": 6675056, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 94120} +{"epoch": 0.80859375, "grad_norm": 0.24756518006324768, "learning_rate": 7.265561527249383e-05, "loss": 0.0023119975812733173, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00231, "step": 207, "tokens/total": 6706624, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 94557} +{"epoch": 0.8125, "grad_norm": 0.2338719666004181, "learning_rate": 7.238823517544436e-05, "loss": 0.003185522509738803, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00319, "step": 208, "tokens/total": 6741008, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 95035} +{"epoch": 0.81640625, "grad_norm": 0.211314395070076, "learning_rate": 7.212013148295333e-05, "loss": 0.0033827372826635838, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00339, "step": 209, "tokens/total": 6774872, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 95517} +{"epoch": 0.8203125, "grad_norm": 0.04923682287335396, "learning_rate": 7.185131535190975e-05, "loss": 0.0003822007856797427, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00038, "step": 210, "tokens/total": 6805816, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 96026} +{"epoch": 0.82421875, "grad_norm": 0.4678303003311157, "learning_rate": 7.158179796885005e-05, "loss": 0.005152560770511627, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00517, "step": 211, "tokens/total": 6837592, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 96477} +{"epoch": 0.828125, "grad_norm": 0.07611093670129776, "learning_rate": 7.131159054949273e-05, "loss": 0.0007398154120892286, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00074, "step": 212, "tokens/total": 6874384, "tokens/train_per_sec_per_gpu": 7.11, "tokens/trainable": 96951} +{"epoch": 0.83203125, "grad_norm": 0.24538062512874603, "learning_rate": 7.104070433827139e-05, "loss": 0.002321182284504175, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00232, "step": 213, "tokens/total": 6905872, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 97350} +{"epoch": 0.8359375, "grad_norm": 0.5418080687522888, "learning_rate": 7.076915060786705e-05, "loss": 0.005303510464727879, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.00532, "step": 214, "tokens/total": 6937984, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 97792} +{"epoch": 0.83984375, "grad_norm": 0.5543075203895569, "learning_rate": 7.049694065873882e-05, "loss": 0.004810214042663574, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00482, "step": 215, "tokens/total": 6970704, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 98257} +{"epoch": 0.84375, "grad_norm": 0.07515121251344681, "learning_rate": 7.022408581865382e-05, "loss": 0.0008246444049291313, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00082, "step": 216, "tokens/total": 7004616, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 98731} +{"epoch": 0.84765625, "grad_norm": 0.34671807289123535, "learning_rate": 6.99505974422157e-05, "loss": 0.002170420251786709, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00217, "step": 217, "tokens/total": 7036880, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 99212} +{"epoch": 0.8515625, "grad_norm": 0.008471252396702766, "learning_rate": 6.967648691039213e-05, "loss": 6.639582716161385e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00007, "step": 218, "tokens/total": 7068208, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 99654} +{"epoch": 0.85546875, "grad_norm": 0.19631044566631317, "learning_rate": 6.940176563004123e-05, "loss": 0.0010948185808956623, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.0011, "step": 219, "tokens/total": 7098112, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 100086} +{"epoch": 0.859375, "grad_norm": 0.31795379519462585, "learning_rate": 6.912644503343682e-05, "loss": 0.004169388208538294, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00418, "step": 220, "tokens/total": 7132432, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 100524} +{"epoch": 0.86328125, "grad_norm": 0.5485131740570068, "learning_rate": 6.885053657779273e-05, "loss": 0.0073966337367892265, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.00742, "step": 221, "tokens/total": 7164016, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 100996} +{"epoch": 0.8671875, "grad_norm": 0.16060703992843628, "learning_rate": 6.857405174478604e-05, "loss": 0.0010755822295323014, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00108, "step": 222, "tokens/total": 7198344, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 101449} +{"epoch": 0.87109375, "grad_norm": 2.059173107147217, "learning_rate": 6.82970020400792e-05, "loss": 0.019824674353003502, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.02002, "step": 223, "tokens/total": 7232192, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 101905} +{"epoch": 0.875, "grad_norm": 0.7925790548324585, "learning_rate": 6.801939899284132e-05, "loss": 0.009008201770484447, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00905, "step": 224, "tokens/total": 7266984, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 102411} +{"epoch": 0.87890625, "grad_norm": 0.6245959997177124, "learning_rate": 6.774125415526827e-05, "loss": 0.0012815927620977163, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00128, "step": 225, "tokens/total": 7299512, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 102882} +{"epoch": 0.8828125, "grad_norm": 0.0007541966042481363, "learning_rate": 6.746257910210214e-05, "loss": 1.941211667144671e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00002, "step": 226, "tokens/total": 7331120, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 103332} +{"epoch": 0.88671875, "grad_norm": 0.1354082077741623, "learning_rate": 6.718338543014937e-05, "loss": 0.0007014721049927175, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.0007, "step": 227, "tokens/total": 7360504, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 103790} +{"epoch": 0.890625, "grad_norm": 0.5079452395439148, "learning_rate": 6.69036847577983e-05, "loss": 0.006149579770863056, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00617, "step": 228, "tokens/total": 7390760, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 104246} +{"epoch": 0.89453125, "grad_norm": 0.3790483772754669, "learning_rate": 6.662348872453553e-05, "loss": 0.0020644385367631912, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00207, "step": 229, "tokens/total": 7423352, "tokens/train_per_sec_per_gpu": 10.51, "tokens/trainable": 104707} +{"epoch": 0.8984375, "grad_norm": 0.04703875258564949, "learning_rate": 6.63428089904618e-05, "loss": 0.00016224136925302446, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00016, "step": 230, "tokens/total": 7457152, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 105167} +{"epoch": 0.90234375, "grad_norm": 0.03571222350001335, "learning_rate": 6.60616572358065e-05, "loss": 0.0002817882050294429, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00028, "step": 231, "tokens/total": 7488648, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 105576} +{"epoch": 0.90625, "grad_norm": 0.2761792838573456, "learning_rate": 6.578004516044172e-05, "loss": 0.00332291959784925, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00333, "step": 232, "tokens/total": 7522160, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 106043} +{"epoch": 0.91015625, "grad_norm": 0.3303437829017639, "learning_rate": 6.549798448339548e-05, "loss": 0.0070444936864078045, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00707, "step": 233, "tokens/total": 7557336, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 106506} +{"epoch": 0.9140625, "grad_norm": 0.3128460645675659, "learning_rate": 6.521548694236384e-05, "loss": 0.0043913512490689754, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.0044, "step": 234, "tokens/total": 7590432, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 106951} +{"epoch": 0.91796875, "grad_norm": 0.2201976776123047, "learning_rate": 6.493256429322259e-05, "loss": 0.0017259400337934494, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00173, "step": 235, "tokens/total": 7623240, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 107382} +{"epoch": 0.921875, "grad_norm": 0.2526547610759735, "learning_rate": 6.464922830953799e-05, "loss": 0.0027879788540303707, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00279, "step": 236, "tokens/total": 7657968, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 107814} +{"epoch": 0.92578125, "grad_norm": 0.5188114643096924, "learning_rate": 6.436549078207688e-05, "loss": 0.011720428243279457, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.01179, "step": 237, "tokens/total": 7689320, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 108274} +{"epoch": 0.9296875, "grad_norm": 0.4764989912509918, "learning_rate": 6.408136351831592e-05, "loss": 0.006160791497677565, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00618, "step": 238, "tokens/total": 7721568, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 108714} +{"epoch": 0.93359375, "grad_norm": 0.01713874563574791, "learning_rate": 6.379685834195036e-05, "loss": 0.00014902935072313994, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00015, "step": 239, "tokens/total": 7756464, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 109220} +{"epoch": 0.9375, "grad_norm": 0.022927720099687576, "learning_rate": 6.351198709240186e-05, "loss": 0.0002721456694416702, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00027, "step": 240, "tokens/total": 7791616, "tokens/train_per_sec_per_gpu": 7.02, "tokens/trainable": 109672} +{"epoch": 0.94140625, "grad_norm": 0.32323798537254333, "learning_rate": 6.32267616243259e-05, "loss": 0.0026845235843211412, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00269, "step": 241, "tokens/total": 7823520, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 110135} +{"epoch": 0.9453125, "grad_norm": 0.2890793979167938, "learning_rate": 6.294119380711849e-05, "loss": 0.001463063177652657, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00146, "step": 242, "tokens/total": 7854424, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 110563} +{"epoch": 0.94921875, "grad_norm": 0.5457670092582703, "learning_rate": 6.265529552442209e-05, "loss": 0.006891611032187939, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00692, "step": 243, "tokens/total": 7887160, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 111049} +{"epoch": 0.953125, "grad_norm": 0.22518502175807953, "learning_rate": 6.236907867363127e-05, "loss": 0.002073091920465231, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00208, "step": 244, "tokens/total": 7918736, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 111495} +{"epoch": 0.95703125, "grad_norm": 1.53590726852417, "learning_rate": 6.208255516539749e-05, "loss": 0.010183682665228844, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.01024, "step": 245, "tokens/total": 7949448, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 111968} +{"epoch": 0.9609375, "grad_norm": 0.24546508491039276, "learning_rate": 6.179573692313344e-05, "loss": 0.0029555540531873703, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00296, "step": 246, "tokens/total": 7984680, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 112416} +{"epoch": 0.96484375, "grad_norm": 0.1797998547554016, "learning_rate": 6.150863588251694e-05, "loss": 0.002759436145424843, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00276, "step": 247, "tokens/total": 8019816, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 112882} +{"epoch": 0.96875, "grad_norm": 0.10661277174949646, "learning_rate": 6.122126399099419e-05, "loss": 0.0007923593511804938, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00079, "step": 248, "tokens/total": 8056272, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 113390} +{"epoch": 0.97265625, "grad_norm": 0.5786979794502258, "learning_rate": 6.0933633207282615e-05, "loss": 0.010734796524047852, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.01079, "step": 249, "tokens/total": 8090224, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 113904} +{"epoch": 0.9765625, "grad_norm": 0.43051204085350037, "learning_rate": 6.064575550087316e-05, "loss": 0.005316047929227352, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00533, "step": 250, "tokens/total": 8124528, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 114351} +{"epoch": 0.98046875, "grad_norm": 0.18971529603004456, "learning_rate": 6.0357642851532245e-05, "loss": 0.0026423432864248753, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00265, "step": 251, "tokens/total": 8159688, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 114842} +{"epoch": 0.984375, "grad_norm": 0.08097948133945465, "learning_rate": 6.0069307248803294e-05, "loss": 0.0005137341795489192, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00051, "step": 252, "tokens/total": 8191800, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 115263} +{"epoch": 0.98828125, "grad_norm": 0.3597053289413452, "learning_rate": 5.9780760691507635e-05, "loss": 0.005986068397760391, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.006, "step": 253, "tokens/total": 8221752, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 115703} +{"epoch": 0.9921875, "grad_norm": 0.12100406736135483, "learning_rate": 5.9492015187245334e-05, "loss": 0.0013363183243200183, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00134, "step": 254, "tokens/total": 8255648, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 116157} +{"epoch": 0.99609375, "grad_norm": 0.385471910238266, "learning_rate": 5.920308275189541e-05, "loss": 0.007686607073992491, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00772, "step": 255, "tokens/total": 8286664, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 116567} +{"epoch": 1.0, "grad_norm": 0.07302063703536987, "learning_rate": 5.8913975409115874e-05, "loss": 0.0012058379361405969, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00121, "step": 256, "tokens/total": 8318368, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 117030} +{"epoch": 1.00390625, "grad_norm": 0.01311566960066557, "learning_rate": 5.8624705189843395e-05, "loss": 0.00013811478856950998, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00014, "step": 257, "tokens/total": 8353336, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 117517} +{"epoch": 1.0078125, "grad_norm": 0.05896522104740143, "learning_rate": 5.833528413179249e-05, "loss": 0.00043564997031353414, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00044, "step": 258, "tokens/total": 8389112, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 118015} +{"epoch": 1.01171875, "grad_norm": 0.13102437555789948, "learning_rate": 5.80457242789548e-05, "loss": 0.0019436449510976672, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00195, "step": 259, "tokens/total": 8426000, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 118520} +{"epoch": 1.015625, "grad_norm": 0.7660783529281616, "learning_rate": 5.77560376810977e-05, "loss": 0.005816752091050148, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.00583, "step": 260, "tokens/total": 8458568, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 118992} +{"epoch": 1.01953125, "grad_norm": 0.04308329522609711, "learning_rate": 5.7466236393263005e-05, "loss": 0.00034656524076126516, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00035, "step": 261, "tokens/total": 8490840, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 119438} +{"epoch": 1.0234375, "grad_norm": 0.12646569311618805, "learning_rate": 5.717633247526522e-05, "loss": 0.001185840112157166, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00119, "step": 262, "tokens/total": 8521792, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 119881} +{"epoch": 1.02734375, "grad_norm": 0.0048444983549416065, "learning_rate": 5.688633799118971e-05, "loss": 5.506083107320592e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00006, "step": 263, "tokens/total": 8551336, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 120285} +{"epoch": 1.03125, "grad_norm": 0.008945104666054249, "learning_rate": 5.659626500889066e-05, "loss": 8.49962088977918e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00008, "step": 264, "tokens/total": 8579872, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 120755} +{"epoch": 1.03515625, "grad_norm": 0.09095881134271622, "learning_rate": 5.6306125599488905e-05, "loss": 0.0008034913917072117, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.0008, "step": 265, "tokens/total": 8608920, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 121194} +{"epoch": 1.0390625, "grad_norm": 0.2665547728538513, "learning_rate": 5.601593183686955e-05, "loss": 0.0014003568794578314, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.0014, "step": 266, "tokens/total": 8643600, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 121670} +{"epoch": 1.04296875, "grad_norm": 0.3224094808101654, "learning_rate": 5.572569579717961e-05, "loss": 0.0051204487681388855, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00513, "step": 267, "tokens/total": 8676032, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 122142} +{"epoch": 1.046875, "grad_norm": 0.0017066330183297396, "learning_rate": 5.543542955832538e-05, "loss": 1.990309829125181e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00002, "step": 268, "tokens/total": 8713240, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 122585} +{"epoch": 1.05078125, "grad_norm": 0.4807676672935486, "learning_rate": 5.514514519946986e-05, "loss": 0.004885909613221884, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.0049, "step": 269, "tokens/total": 8751664, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 123091} +{"epoch": 1.0546875, "grad_norm": 0.18143771588802338, "learning_rate": 5.485485480053015e-05, "loss": 0.0013962223893031478, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.0014, "step": 270, "tokens/total": 8786112, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 123505} +{"epoch": 1.05859375, "grad_norm": 0.2380998283624649, "learning_rate": 5.4564570441674645e-05, "loss": 0.0025808962527662516, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00258, "step": 271, "tokens/total": 8816920, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 123953} +{"epoch": 1.0625, "grad_norm": 0.30481141805648804, "learning_rate": 5.42743042028204e-05, "loss": 0.0027955977711826563, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.0028, "step": 272, "tokens/total": 8851408, "tokens/train_per_sec_per_gpu": 6.81, "tokens/trainable": 124400} +{"epoch": 1.06640625, "grad_norm": 0.2733379602432251, "learning_rate": 5.3984068163130464e-05, "loss": 0.002413226990029216, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00242, "step": 273, "tokens/total": 8883768, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 124870} +{"epoch": 1.0703125, "grad_norm": 1.051348090171814, "learning_rate": 5.369387440051111e-05, "loss": 0.022204149514436722, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.02245, "step": 274, "tokens/total": 8916200, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 125333} +{"epoch": 1.07421875, "grad_norm": 0.02452654205262661, "learning_rate": 5.340373499110935e-05, "loss": 0.00014848173304926604, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00015, "step": 275, "tokens/total": 8949920, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 125834} +{"epoch": 1.078125, "grad_norm": 0.005444176495075226, "learning_rate": 5.3113662008810304e-05, "loss": 3.5796299926005304e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00004, "step": 276, "tokens/total": 8986448, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 126316} +{"epoch": 1.08203125, "grad_norm": 0.019906077533960342, "learning_rate": 5.282366752473479e-05, "loss": 6.331947224680334e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00006, "step": 277, "tokens/total": 9020600, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 126798} +{"epoch": 1.0859375, "grad_norm": 0.10307183861732483, "learning_rate": 5.2533763606737005e-05, "loss": 0.00022259911929722875, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00022, "step": 278, "tokens/total": 9056040, "tokens/train_per_sec_per_gpu": 6.86, "tokens/trainable": 127223} +{"epoch": 1.08984375, "grad_norm": 0.14071044325828552, "learning_rate": 5.224396231890232e-05, "loss": 0.0015680254437029362, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00157, "step": 279, "tokens/total": 9087072, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 127672} +{"epoch": 1.09375, "grad_norm": 0.12417715042829514, "learning_rate": 5.195427572104522e-05, "loss": 0.0014784903032705188, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00148, "step": 280, "tokens/total": 9118376, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 128116} +{"epoch": 1.09765625, "grad_norm": 0.6045622229576111, "learning_rate": 5.166471586820751e-05, "loss": 0.009124290198087692, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00917, "step": 281, "tokens/total": 9152704, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 128589} +{"epoch": 1.1015625, "grad_norm": 0.8433111310005188, "learning_rate": 5.1375294810156615e-05, "loss": 0.008296404033899307, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00833, "step": 282, "tokens/total": 9184448, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 129036} +{"epoch": 1.10546875, "grad_norm": 0.07116912305355072, "learning_rate": 5.1086024590884144e-05, "loss": 0.0009689552243798971, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00097, "step": 283, "tokens/total": 9217392, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 129485} +{"epoch": 1.109375, "grad_norm": 0.048475634306669235, "learning_rate": 5.079691724810461e-05, "loss": 0.000572237535379827, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00057, "step": 284, "tokens/total": 9252016, "tokens/train_per_sec_per_gpu": 7.09, "tokens/trainable": 129920} +{"epoch": 1.11328125, "grad_norm": 0.010713375173509121, "learning_rate": 5.0507984812754684e-05, "loss": 0.00013903039507567883, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00014, "step": 285, "tokens/total": 9284008, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 130417} +{"epoch": 1.1171875, "grad_norm": 0.24087508022785187, "learning_rate": 5.021923930849237e-05, "loss": 0.0025668025482445955, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00257, "step": 286, "tokens/total": 9315984, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 130903} +{"epoch": 1.12109375, "grad_norm": 0.1822972446680069, "learning_rate": 4.99306927511967e-05, "loss": 0.0004895204911008477, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00049, "step": 287, "tokens/total": 9347376, "tokens/train_per_sec_per_gpu": 10.41, "tokens/trainable": 131343} +{"epoch": 1.125, "grad_norm": 0.11919651180505753, "learning_rate": 4.964235714846775e-05, "loss": 0.0015964717604219913, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.0016, "step": 288, "tokens/total": 9377424, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 131763} +{"epoch": 1.12890625, "grad_norm": 0.03795963525772095, "learning_rate": 4.9354244499126866e-05, "loss": 0.0002336824545636773, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00023, "step": 289, "tokens/total": 9408104, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 132229} +{"epoch": 1.1328125, "grad_norm": 0.23070968687534332, "learning_rate": 4.90663667927174e-05, "loss": 0.0014951697085052729, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.0015, "step": 290, "tokens/total": 9439944, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 132678} +{"epoch": 1.13671875, "grad_norm": 0.14841009676456451, "learning_rate": 4.877873600900581e-05, "loss": 0.0016574835171923041, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00166, "step": 291, "tokens/total": 9472104, "tokens/train_per_sec_per_gpu": 6.21, "tokens/trainable": 133136} +{"epoch": 1.140625, "grad_norm": 0.04364839941263199, "learning_rate": 4.849136411748306e-05, "loss": 0.0006345196161419153, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00063, "step": 292, "tokens/total": 9508336, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 133608} +{"epoch": 1.14453125, "grad_norm": 0.14731837809085846, "learning_rate": 4.8204263076866574e-05, "loss": 0.0011924169957637787, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00119, "step": 293, "tokens/total": 9541552, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 134108} +{"epoch": 1.1484375, "grad_norm": 0.11304406821727753, "learning_rate": 4.791744483460251e-05, "loss": 0.0013320090947672725, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00133, "step": 294, "tokens/total": 9575072, "tokens/train_per_sec_per_gpu": 6.56, "tokens/trainable": 134541} +{"epoch": 1.15234375, "grad_norm": 0.13712771236896515, "learning_rate": 4.7630921326368736e-05, "loss": 0.0015649721026420593, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00157, "step": 295, "tokens/total": 9610344, "tokens/train_per_sec_per_gpu": 6.41, "tokens/trainable": 134966} +{"epoch": 1.15625, "grad_norm": 0.1799466907978058, "learning_rate": 4.7344704475577916e-05, "loss": 0.001838606782257557, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00184, "step": 296, "tokens/total": 9639728, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 135402} +{"epoch": 1.16015625, "grad_norm": 0.3172929883003235, "learning_rate": 4.705880619288153e-05, "loss": 0.005720539949834347, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00574, "step": 297, "tokens/total": 9671424, "tokens/train_per_sec_per_gpu": 7.09, "tokens/trainable": 135804} +{"epoch": 1.1640625, "grad_norm": 0.02849152497947216, "learning_rate": 4.677323837567412e-05, "loss": 0.00030726147815585136, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00031, "step": 298, "tokens/total": 9700808, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 136231} +{"epoch": 1.16796875, "grad_norm": 0.06859320402145386, "learning_rate": 4.6488012907598146e-05, "loss": 0.0006029126234352589, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.0006, "step": 299, "tokens/total": 9732312, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 136705} +{"epoch": 1.171875, "grad_norm": 0.08838086575269699, "learning_rate": 4.620314165804964e-05, "loss": 0.0009170416160486639, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00092, "step": 300, "tokens/total": 9764624, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 137178} +{"epoch": 1.17578125, "grad_norm": 0.009998507797718048, "learning_rate": 4.591863648168407e-05, "loss": 9.550770482746884e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.0001, "step": 301, "tokens/total": 9798368, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 137643} +{"epoch": 1.1796875, "grad_norm": 0.22945524752140045, "learning_rate": 4.5634509217923135e-05, "loss": 0.002941780025139451, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00295, "step": 302, "tokens/total": 9830192, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 138078} +{"epoch": 1.18359375, "grad_norm": 0.1602032333612442, "learning_rate": 4.535077169046201e-05, "loss": 0.001225689658895135, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00123, "step": 303, "tokens/total": 9863112, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 138515} +{"epoch": 1.1875, "grad_norm": 0.7816073894500732, "learning_rate": 4.506743570677743e-05, "loss": 0.014298748224973679, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.0144, "step": 304, "tokens/total": 9893192, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 138948} +{"epoch": 1.19140625, "grad_norm": 1.3341065645217896, "learning_rate": 4.478451305763618e-05, "loss": 0.004537603352218866, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00455, "step": 305, "tokens/total": 9925712, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 139408} +{"epoch": 1.1953125, "grad_norm": 0.07823509722948074, "learning_rate": 4.450201551660454e-05, "loss": 0.00047179911052808166, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00047, "step": 306, "tokens/total": 9955736, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 139840} +{"epoch": 1.19921875, "grad_norm": 0.05268227681517601, "learning_rate": 4.4219954839558276e-05, "loss": 0.000411704124417156, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00041, "step": 307, "tokens/total": 9987160, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 140281} +{"epoch": 1.203125, "grad_norm": 0.01796993985772133, "learning_rate": 4.393834276419352e-05, "loss": 0.00010364352056058124, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.0001, "step": 308, "tokens/total": 10017360, "tokens/train_per_sec_per_gpu": 11.14, "tokens/trainable": 140776} +{"epoch": 1.20703125, "grad_norm": 0.0271587036550045, "learning_rate": 4.36571910095382e-05, "loss": 0.00012971644173376262, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00013, "step": 309, "tokens/total": 10052056, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 141228} +{"epoch": 1.2109375, "grad_norm": 0.013242623768746853, "learning_rate": 4.337651127546448e-05, "loss": 8.315574814332649e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00008, "step": 310, "tokens/total": 10085424, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 141679} +{"epoch": 1.21484375, "grad_norm": 0.11297691613435745, "learning_rate": 4.3096315242201736e-05, "loss": 0.0005691410042345524, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00057, "step": 311, "tokens/total": 10116824, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 142122} +{"epoch": 1.21875, "grad_norm": 0.0034611723385751247, "learning_rate": 4.2816614569850635e-05, "loss": 3.257573916926049e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00003, "step": 312, "tokens/total": 10150376, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 142561} +{"epoch": 1.22265625, "grad_norm": 0.25325989723205566, "learning_rate": 4.2537420897897864e-05, "loss": 0.0003773510688915849, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00038, "step": 313, "tokens/total": 10184432, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 143046} +{"epoch": 1.2265625, "grad_norm": 0.033888958394527435, "learning_rate": 4.225874584473174e-05, "loss": 0.0002995567338075489, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.0003, "step": 314, "tokens/total": 10214848, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 143493} +{"epoch": 1.23046875, "grad_norm": 0.006027789320796728, "learning_rate": 4.19806010071587e-05, "loss": 4.2107807530555874e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00004, "step": 315, "tokens/total": 10247952, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 143956} +{"epoch": 1.234375, "grad_norm": 0.012813372537493706, "learning_rate": 4.170299795992081e-05, "loss": 0.00012621114728972316, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00013, "step": 316, "tokens/total": 10280648, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 144411} +{"epoch": 1.23828125, "grad_norm": 0.0742681473493576, "learning_rate": 4.142594825521398e-05, "loss": 0.00035122569533996284, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00035, "step": 317, "tokens/total": 10317712, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 144892} +{"epoch": 1.2421875, "grad_norm": 0.001729971612803638, "learning_rate": 4.114946342220728e-05, "loss": 2.137090268661268e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00002, "step": 318, "tokens/total": 10351632, "tokens/train_per_sec_per_gpu": 6.97, "tokens/trainable": 145339} +{"epoch": 1.24609375, "grad_norm": 0.0025305685121566057, "learning_rate": 4.087355496656321e-05, "loss": 3.213992386008613e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00003, "step": 319, "tokens/total": 10383336, "tokens/train_per_sec_per_gpu": 10.15, "tokens/trainable": 145811} +{"epoch": 1.25, "grad_norm": 0.0422087237238884, "learning_rate": 4.05982343699588e-05, "loss": 0.00020849041175097227, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00021, "step": 320, "tokens/total": 10418016, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 146314} +{"epoch": 1.25390625, "grad_norm": 0.009949288330972195, "learning_rate": 4.0323513089607876e-05, "loss": 6.113090057624504e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00006, "step": 321, "tokens/total": 10451600, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 146781} +{"epoch": 1.2578125, "grad_norm": 0.04362383484840393, "learning_rate": 4.004940255778431e-05, "loss": 0.0001674080704106018, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00017, "step": 322, "tokens/total": 10487568, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 147225} +{"epoch": 1.26171875, "grad_norm": 0.3501319885253906, "learning_rate": 3.977591418134619e-05, "loss": 0.0015369351021945477, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00154, "step": 323, "tokens/total": 10520976, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 147673} +{"epoch": 1.265625, "grad_norm": 0.01552780345082283, "learning_rate": 3.95030593412612e-05, "loss": 0.00015040577272884548, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00015, "step": 324, "tokens/total": 10552776, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 148103} +{"epoch": 1.26953125, "grad_norm": 0.03385813534259796, "learning_rate": 3.923084939213296e-05, "loss": 0.00017573183868080378, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00018, "step": 325, "tokens/total": 10582576, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 148535} +{"epoch": 1.2734375, "grad_norm": 0.01759401522576809, "learning_rate": 3.895929566172861e-05, "loss": 9.057494025910273e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00009, "step": 326, "tokens/total": 10615600, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 148999} +{"epoch": 1.27734375, "grad_norm": 0.2251051813364029, "learning_rate": 3.868840945050728e-05, "loss": 0.0016750060021877289, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00168, "step": 327, "tokens/total": 10650688, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 149431} +{"epoch": 1.28125, "grad_norm": 0.0073803020641207695, "learning_rate": 3.841820203114995e-05, "loss": 8.32182849990204e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00008, "step": 328, "tokens/total": 10681816, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 149886} +{"epoch": 1.28515625, "grad_norm": 0.010879703797399998, "learning_rate": 3.814868464809027e-05, "loss": 9.188025433104485e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00009, "step": 329, "tokens/total": 10711728, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 150288} +{"epoch": 1.2890625, "grad_norm": 0.8226059079170227, "learning_rate": 3.787986851704667e-05, "loss": 0.012381023727357388, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.01246, "step": 330, "tokens/total": 10744952, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 150733} +{"epoch": 1.29296875, "grad_norm": 0.01677289977669716, "learning_rate": 3.7611764824555654e-05, "loss": 0.00011193538375664502, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00011, "step": 331, "tokens/total": 10776192, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 151207} +{"epoch": 1.296875, "grad_norm": 0.004241485148668289, "learning_rate": 3.734438472750619e-05, "loss": 2.4110677259159274e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00002, "step": 332, "tokens/total": 10813336, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 151694} +{"epoch": 1.30078125, "grad_norm": 0.002837607404217124, "learning_rate": 3.707773935267552e-05, "loss": 2.180590854550246e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00002, "step": 333, "tokens/total": 10847808, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 152188} +{"epoch": 1.3046875, "grad_norm": 0.0010325682815164328, "learning_rate": 3.68118397962661e-05, "loss": 1.1875484233314637e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00001, "step": 334, "tokens/total": 10879896, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 152596} +{"epoch": 1.30859375, "grad_norm": 1.1901086568832397, "learning_rate": 3.654669712344384e-05, "loss": 0.015806620940566063, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.01593, "step": 335, "tokens/total": 10908976, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 153052} +{"epoch": 1.3125, "grad_norm": 0.0008912457851693034, "learning_rate": 3.628232236787763e-05, "loss": 1.088732278731186e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00001, "step": 336, "tokens/total": 10943152, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 153491} +{"epoch": 1.31640625, "grad_norm": 0.4584791958332062, "learning_rate": 3.6018726531280144e-05, "loss": 0.003307767678052187, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00331, "step": 337, "tokens/total": 10978184, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 153983} +{"epoch": 1.3203125, "grad_norm": 0.14250101149082184, "learning_rate": 3.575592058295017e-05, "loss": 0.0006636242032982409, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00066, "step": 338, "tokens/total": 11010056, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 154464} +{"epoch": 1.32421875, "grad_norm": 0.03187521919608116, "learning_rate": 3.549391545931585e-05, "loss": 6.826018943684176e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00007, "step": 339, "tokens/total": 11040968, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 154924} +{"epoch": 1.328125, "grad_norm": 0.020824642851948738, "learning_rate": 3.5232722063479914e-05, "loss": 0.00016181297542061657, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00016, "step": 340, "tokens/total": 11073424, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 155373} +{"epoch": 1.33203125, "grad_norm": 0.023208793252706528, "learning_rate": 3.49723512647657e-05, "loss": 0.00033657235326245427, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00034, "step": 341, "tokens/total": 11109224, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 155873} +{"epoch": 1.3359375, "grad_norm": 0.040250957012176514, "learning_rate": 3.471281389826491e-05, "loss": 9.644995589042082e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.0001, "step": 342, "tokens/total": 11139856, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 156278} +{"epoch": 1.33984375, "grad_norm": 0.15981747210025787, "learning_rate": 3.4454120764386764e-05, "loss": 0.001154238940216601, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00115, "step": 343, "tokens/total": 11175224, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 156733} +{"epoch": 1.34375, "grad_norm": 0.0037592733278870583, "learning_rate": 3.4196282628408526e-05, "loss": 4.58205868199002e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00005, "step": 344, "tokens/total": 11207112, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 157203} +{"epoch": 1.34765625, "grad_norm": 0.004076949320733547, "learning_rate": 3.3939310220027456e-05, "loss": 7.070083665894344e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00007, "step": 345, "tokens/total": 11240432, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 157707} +{"epoch": 1.3515625, "grad_norm": 0.006908221170306206, "learning_rate": 3.3683214232914404e-05, "loss": 1.7060992831829935e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00002, "step": 346, "tokens/total": 11274584, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 158180} +{"epoch": 1.35546875, "grad_norm": 0.03559071198105812, "learning_rate": 3.342800532426873e-05, "loss": 0.000273198529612273, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00027, "step": 347, "tokens/total": 11307376, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 158646} +{"epoch": 1.359375, "grad_norm": 0.17426390945911407, "learning_rate": 3.317369411437484e-05, "loss": 0.0015962847974151373, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.0016, "step": 348, "tokens/total": 11340640, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 159115} +{"epoch": 1.36328125, "grad_norm": 0.03259862959384918, "learning_rate": 3.292029118616024e-05, "loss": 0.00039978689164854586, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.0004, "step": 349, "tokens/total": 11371632, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 159538} +{"epoch": 1.3671875, "grad_norm": 0.18651357293128967, "learning_rate": 3.266780708475511e-05, "loss": 0.002337759593501687, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00234, "step": 350, "tokens/total": 11404248, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 160016} +{"epoch": 1.37109375, "grad_norm": 0.019524402916431427, "learning_rate": 3.241625231705354e-05, "loss": 0.00010522563388803974, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00011, "step": 351, "tokens/total": 11439552, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 160475} +{"epoch": 1.375, "grad_norm": 0.012347620911896229, "learning_rate": 3.216563735127618e-05, "loss": 0.00011971787898801267, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00012, "step": 352, "tokens/total": 11470944, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 160952} +{"epoch": 1.37890625, "grad_norm": 0.2471960037946701, "learning_rate": 3.191597261653475e-05, "loss": 0.001952206832356751, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00195, "step": 353, "tokens/total": 11506304, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 161420} +{"epoch": 1.3828125, "grad_norm": 2.6456613540649414, "learning_rate": 3.166726850239794e-05, "loss": 0.0011997036635875702, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.0012, "step": 354, "tokens/total": 11535424, "tokens/train_per_sec_per_gpu": 11.0, "tokens/trainable": 161868} +{"epoch": 1.38671875, "grad_norm": 0.0027058266568928957, "learning_rate": 3.141953535845912e-05, "loss": 2.471780317137018e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00002, "step": 355, "tokens/total": 11565176, "tokens/train_per_sec_per_gpu": 10.26, "tokens/trainable": 162318} +{"epoch": 1.390625, "grad_norm": 0.226266548037529, "learning_rate": 3.11727834939056e-05, "loss": 0.0008792744483798742, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00088, "step": 356, "tokens/total": 11595528, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 162789} +{"epoch": 1.39453125, "grad_norm": 0.5570091009140015, "learning_rate": 3.092702317708967e-05, "loss": 0.010999402031302452, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.01106, "step": 357, "tokens/total": 11627248, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 163254} +{"epoch": 1.3984375, "grad_norm": 0.09474620223045349, "learning_rate": 3.0682264635101276e-05, "loss": 0.0008793296874500811, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00088, "step": 358, "tokens/total": 11659440, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 163715} +{"epoch": 1.40234375, "grad_norm": 0.11742257326841354, "learning_rate": 3.0438518053342407e-05, "loss": 0.000552443612832576, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00055, "step": 359, "tokens/total": 11693288, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 164197} +{"epoch": 1.40625, "grad_norm": 0.2086295336484909, "learning_rate": 3.0195793575103266e-05, "loss": 0.002380533143877983, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00238, "step": 360, "tokens/total": 11730888, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 164661} +{"epoch": 1.41015625, "grad_norm": 0.013920009136199951, "learning_rate": 2.9954101301140146e-05, "loss": 0.00011457806249381974, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00011, "step": 361, "tokens/total": 11765656, "tokens/train_per_sec_per_gpu": 6.89, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.05254616588354111, "learning_rate": 2.9713451289255123e-05, "loss": 0.0004433058784343302, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00044, "step": 362, "tokens/total": 11793456, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 165552} +{"epoch": 1.41796875, "grad_norm": 0.0018456476973369718, "learning_rate": 2.9473853553877484e-05, "loss": 2.580435830168426e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00003, "step": 363, "tokens/total": 11826280, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 166026} +{"epoch": 1.421875, "grad_norm": 0.09761562198400497, "learning_rate": 2.9235318065647e-05, "loss": 0.0006601369823329151, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00066, "step": 364, "tokens/total": 11863552, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 166486} +{"epoch": 1.42578125, "grad_norm": 0.0007161365356296301, "learning_rate": 2.8997854750998964e-05, "loss": 1.0992088391503785e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00001, "step": 365, "tokens/total": 11898616, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 166959} +{"epoch": 1.4296875, "grad_norm": 0.007225404493510723, "learning_rate": 2.8761473491751258e-05, "loss": 3.938953159376979e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00004, "step": 366, "tokens/total": 11930728, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 167394} +{"epoch": 1.43359375, "grad_norm": 0.004835801664739847, "learning_rate": 2.8526184124692883e-05, "loss": 4.931920193484984e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00005, "step": 367, "tokens/total": 11964976, "tokens/train_per_sec_per_gpu": 6.4, "tokens/trainable": 167832} +{"epoch": 1.4375, "grad_norm": 0.018557650968432426, "learning_rate": 2.829199644117484e-05, "loss": 0.00011596402327995747, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00012, "step": 368, "tokens/total": 11996696, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 168242} +{"epoch": 1.44140625, "grad_norm": 0.02286013774573803, "learning_rate": 2.8058920186702553e-05, "loss": 0.00022500351769849658, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00023, "step": 369, "tokens/total": 12033432, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 168715} +{"epoch": 1.4453125, "grad_norm": 0.48172977566719055, "learning_rate": 2.782696506053033e-05, "loss": 0.008261239156126976, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.0083, "step": 370, "tokens/total": 12067616, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 169147} +{"epoch": 1.44921875, "grad_norm": 0.8992995023727417, "learning_rate": 2.7596140715257824e-05, "loss": 0.009562648832798004, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00961, "step": 371, "tokens/total": 12099008, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 169628} +{"epoch": 1.453125, "grad_norm": 0.0028636560309678316, "learning_rate": 2.7366456756428184e-05, "loss": 4.135803101235069e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00004, "step": 372, "tokens/total": 12131304, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 170085} +{"epoch": 1.45703125, "grad_norm": 0.24486090242862701, "learning_rate": 2.7137922742128486e-05, "loss": 0.0018681498477235436, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00187, "step": 373, "tokens/total": 12164104, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 170584} +{"epoch": 1.4609375, "grad_norm": 0.0038282740861177444, "learning_rate": 2.691054818259188e-05, "loss": 3.972392369178124e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00004, "step": 374, "tokens/total": 12197776, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 171058} +{"epoch": 1.46484375, "grad_norm": 0.7020986676216125, "learning_rate": 2.6684342539801933e-05, "loss": 0.005676680710166693, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00569, "step": 375, "tokens/total": 12228064, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 171518} +{"epoch": 1.46875, "grad_norm": 0.006726031191647053, "learning_rate": 2.645931522709877e-05, "loss": 6.188623228808865e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00006, "step": 376, "tokens/total": 12261688, "tokens/train_per_sec_per_gpu": 10.72, "tokens/trainable": 172007} +{"epoch": 1.47265625, "grad_norm": 0.03776044398546219, "learning_rate": 2.6235475608787365e-05, "loss": 0.00037066437653265893, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00037, "step": 377, "tokens/total": 12294240, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 172468} +{"epoch": 1.4765625, "grad_norm": 0.3361227512359619, "learning_rate": 2.6012832999747916e-05, "loss": 0.002179669449105859, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00218, "step": 378, "tokens/total": 12324528, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 172923} +{"epoch": 1.48046875, "grad_norm": 0.4022933840751648, "learning_rate": 2.579139666504821e-05, "loss": 0.007893390953540802, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00792, "step": 379, "tokens/total": 12360288, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 173370} +{"epoch": 1.484375, "grad_norm": 0.05767251178622246, "learning_rate": 2.557117581955798e-05, "loss": 0.00029326981166377664, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00029, "step": 380, "tokens/total": 12391216, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 173837} +{"epoch": 1.48828125, "grad_norm": 0.5231335759162903, "learning_rate": 2.5352179627565532e-05, "loss": 0.004635948222130537, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00465, "step": 381, "tokens/total": 12423624, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 174294} +{"epoch": 1.4921875, "grad_norm": 0.34918323159217834, "learning_rate": 2.5134417202396277e-05, "loss": 0.0034225943963974714, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00343, "step": 382, "tokens/total": 12456032, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 174701} +{"epoch": 1.49609375, "grad_norm": 0.15589545667171478, "learning_rate": 2.491789760603361e-05, "loss": 0.00204786891117692, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00205, "step": 383, "tokens/total": 12489336, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 175131} +{"epoch": 1.5, "grad_norm": 0.21111702919006348, "learning_rate": 2.4702629848741764e-05, "loss": 0.002008893759921193, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00201, "step": 384, "tokens/total": 12521944, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 175586} +{"epoch": 1.50390625, "grad_norm": 0.35749560594558716, "learning_rate": 2.4488622888690785e-05, "loss": 0.005422234535217285, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00544, "step": 385, "tokens/total": 12554024, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 176026} +{"epoch": 1.5078125, "grad_norm": 0.07194087654352188, "learning_rate": 2.427588563158384e-05, "loss": 0.0006966604269109666, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.0007, "step": 386, "tokens/total": 12587104, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 176512} +{"epoch": 1.51171875, "grad_norm": 0.21252404153347015, "learning_rate": 2.406442693028651e-05, "loss": 0.001272004097700119, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00127, "step": 387, "tokens/total": 12617456, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 176943} +{"epoch": 1.515625, "grad_norm": 0.03155597299337387, "learning_rate": 2.3854255584458547e-05, "loss": 0.00015958992298692465, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00016, "step": 388, "tokens/total": 12652696, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 177370} +{"epoch": 1.51953125, "grad_norm": 0.038278743624687195, "learning_rate": 2.3645380340187508e-05, "loss": 0.0001688457850832492, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00017, "step": 389, "tokens/total": 12685792, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 177851} +{"epoch": 1.5234375, "grad_norm": 0.08103705197572708, "learning_rate": 2.3437809889624914e-05, "loss": 0.000579597894102335, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00058, "step": 390, "tokens/total": 12716120, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 178310} +{"epoch": 1.52734375, "grad_norm": 0.04301515966653824, "learning_rate": 2.3231552870624487e-05, "loss": 0.00022881121549289674, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00023, "step": 391, "tokens/total": 12746216, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 178736} +{"epoch": 1.53125, "grad_norm": 0.06369390338659286, "learning_rate": 2.3026617866382657e-05, "loss": 0.0003568372339941561, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00036, "step": 392, "tokens/total": 12778592, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 179233} +{"epoch": 1.53515625, "grad_norm": 0.00784015841782093, "learning_rate": 2.2823013405081507e-05, "loss": 0.00011141373397549614, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00011, "step": 393, "tokens/total": 12812008, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 179730} +{"epoch": 1.5390625, "grad_norm": 0.1471669226884842, "learning_rate": 2.2620747959533722e-05, "loss": 0.0026706038042902946, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00267, "step": 394, "tokens/total": 12844264, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 180227} +{"epoch": 1.54296875, "grad_norm": 0.05156897008419037, "learning_rate": 2.2419829946830123e-05, "loss": 0.00030581411556340754, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00031, "step": 395, "tokens/total": 12878328, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 180687} +{"epoch": 1.546875, "grad_norm": 0.04716164991259575, "learning_rate": 2.2220267727989325e-05, "loss": 0.0005423688562586904, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00054, "step": 396, "tokens/total": 12907912, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 181141} +{"epoch": 1.55078125, "grad_norm": 0.08171311020851135, "learning_rate": 2.202206960760984e-05, "loss": 0.000893247895874083, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00089, "step": 397, "tokens/total": 12941968, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 181648} +{"epoch": 1.5546875, "grad_norm": 0.056566089391708374, "learning_rate": 2.182524383352446e-05, "loss": 0.0005396052147261798, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00054, "step": 398, "tokens/total": 12975240, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 182109} +{"epoch": 1.55859375, "grad_norm": 0.19907744228839874, "learning_rate": 2.1629798596457056e-05, "loss": 0.0018499899888411164, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00185, "step": 399, "tokens/total": 13006912, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 182559} +{"epoch": 1.5625, "grad_norm": 0.08316292613744736, "learning_rate": 2.1435742029681725e-05, "loss": 0.000636445649433881, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00064, "step": 400, "tokens/total": 13037640, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 182985} +{"epoch": 1.56640625, "grad_norm": 0.0025031755212694407, "learning_rate": 2.124308220868431e-05, "loss": 3.923914482584223e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00004, "step": 401, "tokens/total": 13070608, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 183440} +{"epoch": 1.5703125, "grad_norm": 0.3469947278499603, "learning_rate": 2.105182715082638e-05, "loss": 0.0004951324663124979, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.0005, "step": 402, "tokens/total": 13101784, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 183917} +{"epoch": 1.57421875, "grad_norm": 0.019016152247786522, "learning_rate": 2.0861984815011552e-05, "loss": 0.00017254333943128586, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00017, "step": 403, "tokens/total": 13133528, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 184348} +{"epoch": 1.578125, "grad_norm": 0.04289346560835838, "learning_rate": 2.0673563101354323e-05, "loss": 0.0003860235447064042, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00039, "step": 404, "tokens/total": 13167088, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 184786} +{"epoch": 1.58203125, "grad_norm": 0.043170154094696045, "learning_rate": 2.0486569850851317e-05, "loss": 0.00040218167123384774, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.0004, "step": 405, "tokens/total": 13200392, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 185249} +{"epoch": 1.5859375, "grad_norm": 0.0053694890812039375, "learning_rate": 2.0301012845054956e-05, "loss": 8.83910252014175e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00009, "step": 406, "tokens/total": 13235488, "tokens/train_per_sec_per_gpu": 6.87, "tokens/trainable": 185680} +{"epoch": 1.58984375, "grad_norm": 0.05239906907081604, "learning_rate": 2.011689980574966e-05, "loss": 0.00032087945146486163, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00032, "step": 407, "tokens/total": 13269416, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 186100} +{"epoch": 1.59375, "grad_norm": 0.27311840653419495, "learning_rate": 1.993423839463052e-05, "loss": 0.004385136533528566, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00439, "step": 408, "tokens/total": 13301360, "tokens/train_per_sec_per_gpu": 10.42, "tokens/trainable": 186565} +{"epoch": 1.59765625, "grad_norm": 0.015250412747263908, "learning_rate": 1.975303621298445e-05, "loss": 0.00014830166765023023, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00015, "step": 409, "tokens/total": 13336392, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 187017} +{"epoch": 1.6015625, "grad_norm": 0.07383466511964798, "learning_rate": 1.957330080137385e-05, "loss": 0.0003911509702447802, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00039, "step": 410, "tokens/total": 13366072, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 187468} +{"epoch": 1.60546875, "grad_norm": 0.2908157706260681, "learning_rate": 1.9395039639322864e-05, "loss": 0.0016693116631358862, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00167, "step": 411, "tokens/total": 13400832, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 187961} +{"epoch": 1.609375, "grad_norm": 0.007157411426305771, "learning_rate": 1.9218260145006073e-05, "loss": 5.6072702136589214e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00006, "step": 412, "tokens/total": 13434920, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 188447} +{"epoch": 1.61328125, "grad_norm": 0.010192830115556717, "learning_rate": 1.904296967493982e-05, "loss": 0.00010952028242172673, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00011, "step": 413, "tokens/total": 13466512, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 188903} +{"epoch": 1.6171875, "grad_norm": 0.13562174141407013, "learning_rate": 1.8869175523676064e-05, "loss": 0.001753663644194603, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00176, "step": 414, "tokens/total": 13500352, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 189398} +{"epoch": 1.62109375, "grad_norm": 0.08840803802013397, "learning_rate": 1.869688492349885e-05, "loss": 0.0005758318002335727, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00058, "step": 415, "tokens/total": 13532448, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 189850} +{"epoch": 1.625, "grad_norm": 0.2512092590332031, "learning_rate": 1.85261050441233e-05, "loss": 0.001931329956278205, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00193, "step": 416, "tokens/total": 13565648, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 190306} +{"epoch": 1.62890625, "grad_norm": 0.3142760694026947, "learning_rate": 1.8356842992397304e-05, "loss": 0.005915830843150616, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00593, "step": 417, "tokens/total": 13597600, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 190746} +{"epoch": 1.6328125, "grad_norm": 0.14049586653709412, "learning_rate": 1.8189105812005714e-05, "loss": 0.0011706833029165864, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00117, "step": 418, "tokens/total": 13630456, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 191156} +{"epoch": 1.63671875, "grad_norm": 0.013081172481179237, "learning_rate": 1.802290048317732e-05, "loss": 9.259879152523354e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00009, "step": 419, "tokens/total": 13660896, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 191573} +{"epoch": 1.640625, "grad_norm": 0.01800309680402279, "learning_rate": 1.785823392239424e-05, "loss": 0.0001771900278981775, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00018, "step": 420, "tokens/total": 13693176, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 192014} +{"epoch": 1.64453125, "grad_norm": 0.0024995480198413134, "learning_rate": 1.7695112982104225e-05, "loss": 3.461617598077282e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00003, "step": 421, "tokens/total": 13725400, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 192453} +{"epoch": 1.6484375, "grad_norm": 0.03527259826660156, "learning_rate": 1.7533544450435433e-05, "loss": 0.00012509478256106377, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00013, "step": 422, "tokens/total": 13752752, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 192886} +{"epoch": 1.65234375, "grad_norm": 0.002378916135057807, "learning_rate": 1.7373535050913946e-05, "loss": 2.887290065700654e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00003, "step": 423, "tokens/total": 13786136, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 193323} +{"epoch": 1.65625, "grad_norm": 0.4140840172767639, "learning_rate": 1.721509144218405e-05, "loss": 0.002726626116782427, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00273, "step": 424, "tokens/total": 13815904, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 193753} +{"epoch": 1.66015625, "grad_norm": 0.013823933899402618, "learning_rate": 1.705822021773101e-05, "loss": 0.00011699306196533144, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00012, "step": 425, "tokens/total": 13847976, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 194240} +{"epoch": 1.6640625, "grad_norm": 0.29887625575065613, "learning_rate": 1.69029279056068e-05, "loss": 0.0021728361025452614, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00218, "step": 426, "tokens/total": 13883200, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 194680} +{"epoch": 1.66796875, "grad_norm": 0.02917727828025818, "learning_rate": 1.6749220968158415e-05, "loss": 0.00016229719039984047, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00016, "step": 427, "tokens/total": 13916080, "tokens/train_per_sec_per_gpu": 10.34, "tokens/trainable": 195186} +{"epoch": 1.671875, "grad_norm": 0.03129657730460167, "learning_rate": 1.659710580175893e-05, "loss": 0.00019593267643358558, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.0002, "step": 428, "tokens/total": 13946448, "tokens/train_per_sec_per_gpu": 10.13, "tokens/trainable": 195693} +{"epoch": 1.67578125, "grad_norm": 0.00843913946300745, "learning_rate": 1.644658873654133e-05, "loss": 0.00010842936171684414, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00011, "step": 429, "tokens/total": 13975696, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 196130} +{"epoch": 1.6796875, "grad_norm": 0.0032142302952706814, "learning_rate": 1.629767603613508e-05, "loss": 4.06644721806515e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00004, "step": 430, "tokens/total": 14002616, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 196530} +{"epoch": 1.68359375, "grad_norm": 0.007689748890697956, "learning_rate": 1.615037389740547e-05, "loss": 9.293916082242504e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00009, "step": 431, "tokens/total": 14034848, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 196974} +{"epoch": 1.6875, "grad_norm": 0.025767913088202477, "learning_rate": 1.600468845019576e-05, "loss": 0.00015222324873320758, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00015, "step": 432, "tokens/total": 14069032, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 197454} +{"epoch": 1.69140625, "grad_norm": 0.00183839350938797, "learning_rate": 1.5860625757072092e-05, "loss": 1.966219861060381e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00002, "step": 433, "tokens/total": 14101296, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 197902} +{"epoch": 1.6953125, "grad_norm": 0.0072134751826524734, "learning_rate": 1.571819181307116e-05, "loss": 6.623464287258685e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00007, "step": 434, "tokens/total": 14135880, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 198374} +{"epoch": 1.69921875, "grad_norm": 0.014135426841676235, "learning_rate": 1.557739254545075e-05, "loss": 0.0001567206927575171, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00016, "step": 435, "tokens/total": 14166040, "tokens/train_per_sec_per_gpu": 10.61, "tokens/trainable": 198846} +{"epoch": 1.703125, "grad_norm": 0.11603060364723206, "learning_rate": 1.543823381344311e-05, "loss": 0.0007414940046146512, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00074, "step": 436, "tokens/total": 14196032, "tokens/train_per_sec_per_gpu": 10.74, "tokens/trainable": 199320} +{"epoch": 1.70703125, "grad_norm": 0.025129206478595734, "learning_rate": 1.5300721408011114e-05, "loss": 0.0002094765950459987, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00021, "step": 437, "tokens/total": 14228008, "tokens/train_per_sec_per_gpu": 10.22, "tokens/trainable": 199777} +{"epoch": 1.7109375, "grad_norm": 0.028187206014990807, "learning_rate": 1.5164861051607254e-05, "loss": 0.00015124297351576388, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.76, "memory/max_allocated (GiB)": 65.76, "ppl": 1.00015, "step": 438, "tokens/total": 14261744, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 200194} +{"epoch": 1.71484375, "grad_norm": 0.0017564555164426565, "learning_rate": 1.5030658397935521e-05, "loss": 2.4558958102716133e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00002, "step": 439, "tokens/total": 14295208, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 200663} +{"epoch": 1.71875, "grad_norm": 0.04521961510181427, "learning_rate": 1.4898119031716104e-05, "loss": 0.0002631057286635041, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00026, "step": 440, "tokens/total": 14327264, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 201121} +{"epoch": 1.72265625, "grad_norm": 0.05111982673406601, "learning_rate": 1.476724846845306e-05, "loss": 0.0003598073963075876, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00036, "step": 441, "tokens/total": 14363088, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 201598} +{"epoch": 1.7265625, "grad_norm": 0.14766232669353485, "learning_rate": 1.463805215420471e-05, "loss": 0.0002769470738712698, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00028, "step": 442, "tokens/total": 14396088, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 202068} +{"epoch": 1.73046875, "grad_norm": 0.002372645540162921, "learning_rate": 1.451053546535705e-05, "loss": 2.635215423651971e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00003, "step": 443, "tokens/total": 14429232, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 202524} +{"epoch": 1.734375, "grad_norm": 0.0013990586157888174, "learning_rate": 1.438470370840001e-05, "loss": 2.176586895075161e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00002, "step": 444, "tokens/total": 14463264, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 202996} +{"epoch": 1.73828125, "grad_norm": 0.0013517123879864812, "learning_rate": 1.4260562119706606e-05, "loss": 1.9849821910611354e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00002, "step": 445, "tokens/total": 14493552, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 203470} +{"epoch": 1.7421875, "grad_norm": 0.016206908971071243, "learning_rate": 1.413811586531508e-05, "loss": 0.0001278911076951772, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00013, "step": 446, "tokens/total": 14525240, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 203919} +{"epoch": 1.74609375, "grad_norm": 0.12385941296815872, "learning_rate": 1.4017370040713884e-05, "loss": 0.0009904676117002964, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00099, "step": 447, "tokens/total": 14559560, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 204376} +{"epoch": 1.75, "grad_norm": 0.1340293139219284, "learning_rate": 1.3898329670629645e-05, "loss": 0.0009032065863721073, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.0009, "step": 448, "tokens/total": 14594160, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 204821} +{"epoch": 1.75390625, "grad_norm": 0.0006050472147762775, "learning_rate": 1.3780999708818058e-05, "loss": 1.0895000741584226e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00001, "step": 449, "tokens/total": 14624224, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 205271} +{"epoch": 1.7578125, "grad_norm": 0.004003340844064951, "learning_rate": 1.3665385037857758e-05, "loss": 2.8602289603441022e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00003, "step": 450, "tokens/total": 14654984, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 205723} +{"epoch": 1.76171875, "grad_norm": 0.011257242411375046, "learning_rate": 1.3551490468947126e-05, "loss": 7.668880425626412e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00008, "step": 451, "tokens/total": 14685440, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 206169} +{"epoch": 1.765625, "grad_norm": 0.4732505679130554, "learning_rate": 1.3439320741704075e-05, "loss": 0.0004928318085148931, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00049, "step": 452, "tokens/total": 14716120, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 206592} +{"epoch": 1.76953125, "grad_norm": 0.05080113932490349, "learning_rate": 1.3328880523968808e-05, "loss": 0.000520411878824234, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00052, "step": 453, "tokens/total": 14748424, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 207055} +{"epoch": 1.7734375, "grad_norm": 0.0021125555504113436, "learning_rate": 1.3220174411609587e-05, "loss": 2.66527622443391e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00003, "step": 454, "tokens/total": 14781256, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 207494} +{"epoch": 1.77734375, "grad_norm": 0.04373720660805702, "learning_rate": 1.3113206928331471e-05, "loss": 0.0004127855645492673, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00041, "step": 455, "tokens/total": 14809552, "tokens/train_per_sec_per_gpu": 9.88, "tokens/trainable": 207956} +{"epoch": 1.78125, "grad_norm": 0.0025095257442444563, "learning_rate": 1.300798252548806e-05, "loss": 2.6917261493508704e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00003, "step": 456, "tokens/total": 14841808, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 208408} +{"epoch": 1.78515625, "grad_norm": 0.000707295082975179, "learning_rate": 1.2904505581896265e-05, "loss": 1.3526510883821175e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00001, "step": 457, "tokens/total": 14875896, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 208889} +{"epoch": 1.7890625, "grad_norm": 0.0026375458110123873, "learning_rate": 1.2802780403654082e-05, "loss": 4.133467882638797e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00004, "step": 458, "tokens/total": 14908424, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 209336} +{"epoch": 1.79296875, "grad_norm": 0.017334599047899246, "learning_rate": 1.2702811223961408e-05, "loss": 4.185618308838457e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00004, "step": 459, "tokens/total": 14939168, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 209797} +{"epoch": 1.796875, "grad_norm": 0.020696308463811874, "learning_rate": 1.2604602202943861e-05, "loss": 0.00013204840070102364, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00013, "step": 460, "tokens/total": 14969872, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 210240} +{"epoch": 1.80078125, "grad_norm": 0.14669598639011383, "learning_rate": 1.2508157427479686e-05, "loss": 0.0003929344820789993, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00039, "step": 461, "tokens/total": 15001472, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 210702} +{"epoch": 1.8046875, "grad_norm": 0.16724318265914917, "learning_rate": 1.2413480911029655e-05, "loss": 0.0011153651867061853, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00112, "step": 462, "tokens/total": 15030592, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 211140} +{"epoch": 1.80859375, "grad_norm": 0.0309348963201046, "learning_rate": 1.2320576593470082e-05, "loss": 0.00013658852549269795, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00014, "step": 463, "tokens/total": 15063648, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 211575} +{"epoch": 1.8125, "grad_norm": 0.03844481706619263, "learning_rate": 1.2229448340928828e-05, "loss": 0.0002539432607591152, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00025, "step": 464, "tokens/total": 15098248, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 212083} +{"epoch": 1.81640625, "grad_norm": 0.0006312375771813095, "learning_rate": 1.2140099945624458e-05, "loss": 1.1229134543100372e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00001, "step": 465, "tokens/total": 15133216, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 212558} +{"epoch": 1.8203125, "grad_norm": 0.006687242072075605, "learning_rate": 1.205253512570841e-05, "loss": 2.5767718398128636e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00003, "step": 466, "tokens/total": 15165416, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 213014} +{"epoch": 1.82421875, "grad_norm": 0.013255500234663486, "learning_rate": 1.1966757525110255e-05, "loss": 0.00015365284343715757, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00015, "step": 467, "tokens/total": 15193688, "tokens/train_per_sec_per_gpu": 10.09, "tokens/trainable": 213465} +{"epoch": 1.828125, "grad_norm": 0.36177581548690796, "learning_rate": 1.1882770713386095e-05, "loss": 0.008368177339434624, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.0084, "step": 468, "tokens/total": 15228600, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 213940} +{"epoch": 1.83203125, "grad_norm": 0.07191647589206696, "learning_rate": 1.180057818556998e-05, "loss": 0.0006436024559661746, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00064, "step": 469, "tokens/total": 15262192, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 214415} +{"epoch": 1.8359375, "grad_norm": 0.00296442280523479, "learning_rate": 1.1720183362028494e-05, "loss": 1.2375670848996378e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00001, "step": 470, "tokens/total": 15294920, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 214880} +{"epoch": 1.83984375, "grad_norm": 0.1524697095155716, "learning_rate": 1.1641589588318387e-05, "loss": 0.0005696783191524446, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00057, "step": 471, "tokens/total": 15327432, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 215334} +{"epoch": 1.84375, "grad_norm": 0.0037736722733825445, "learning_rate": 1.1564800135047418e-05, "loss": 3.8075890188338235e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00004, "step": 472, "tokens/total": 15359928, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 215822} +{"epoch": 1.84765625, "grad_norm": 0.00334280077368021, "learning_rate": 1.148981819773816e-05, "loss": 1.880635682027787e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00002, "step": 473, "tokens/total": 15391400, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 216239} +{"epoch": 1.8515625, "grad_norm": 0.008249087259173393, "learning_rate": 1.1416646896695086e-05, "loss": 3.063416079385206e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00003, "step": 474, "tokens/total": 15424144, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 216691} +{"epoch": 1.85546875, "grad_norm": 0.02139686979353428, "learning_rate": 1.1345289276874717e-05, "loss": 0.00015738011279609054, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00016, "step": 475, "tokens/total": 15457680, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 217162} +{"epoch": 1.859375, "grad_norm": 0.00631346320733428, "learning_rate": 1.1275748307758873e-05, "loss": 1.3788140677206684e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00001, "step": 476, "tokens/total": 15492056, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 217638} +{"epoch": 1.86328125, "grad_norm": 0.0004053537268191576, "learning_rate": 1.1208026883231147e-05, "loss": 7.863444807298947e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00001, "step": 477, "tokens/total": 15521544, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 218053} +{"epoch": 1.8671875, "grad_norm": 0.005711439996957779, "learning_rate": 1.1142127821456433e-05, "loss": 3.213567106286064e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00003, "step": 478, "tokens/total": 15553320, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 218503} +{"epoch": 1.87109375, "grad_norm": 0.007815453223884106, "learning_rate": 1.1078053864763674e-05, "loss": 6.176311580929905e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00006, "step": 479, "tokens/total": 15586208, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 218964} +{"epoch": 1.875, "grad_norm": 0.4135259985923767, "learning_rate": 1.1015807679531756e-05, "loss": 0.0019950633868575096, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.002, "step": 480, "tokens/total": 15616432, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 219412} +{"epoch": 1.87890625, "grad_norm": 0.0055697523057460785, "learning_rate": 1.0955391856078528e-05, "loss": 4.3634441681206226e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00004, "step": 481, "tokens/total": 15650056, "tokens/train_per_sec_per_gpu": 10.29, "tokens/trainable": 219886} +{"epoch": 1.8828125, "grad_norm": 0.06293992698192596, "learning_rate": 1.0896808908553007e-05, "loss": 0.0003916235582437366, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00039, "step": 482, "tokens/total": 15680336, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 220343} +{"epoch": 1.88671875, "grad_norm": 0.019533822312951088, "learning_rate": 1.0840061274830763e-05, "loss": 0.00023925320419948548, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00024, "step": 483, "tokens/total": 15711552, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 220825} +{"epoch": 1.890625, "grad_norm": 0.0019123479723930359, "learning_rate": 1.0785151316412473e-05, "loss": 1.93003306776518e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00002, "step": 484, "tokens/total": 15744664, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 221282} +{"epoch": 1.89453125, "grad_norm": 0.10812956839799881, "learning_rate": 1.0732081318325639e-05, "loss": 0.00040224328404292464, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.0004, "step": 485, "tokens/total": 15782800, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 221780} +{"epoch": 1.8984375, "grad_norm": 0.24160876870155334, "learning_rate": 1.0680853489029501e-05, "loss": 0.0018462500302121043, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00185, "step": 486, "tokens/total": 15817368, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 222210} +{"epoch": 1.90234375, "grad_norm": 0.007703368552029133, "learning_rate": 1.0631469960323152e-05, "loss": 3.7138175684958696e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00004, "step": 487, "tokens/total": 15850408, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 222646} +{"epoch": 1.90625, "grad_norm": 0.12332645058631897, "learning_rate": 1.0583932787256783e-05, "loss": 0.0008310020202770829, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00083, "step": 488, "tokens/total": 15881448, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 223066} +{"epoch": 1.91015625, "grad_norm": 0.007130345795303583, "learning_rate": 1.0538243948046206e-05, "loss": 4.9392729124519974e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00005, "step": 489, "tokens/total": 15913736, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 223512} +{"epoch": 1.9140625, "grad_norm": 0.029169492423534393, "learning_rate": 1.0494405343990523e-05, "loss": 0.00015738508955109864, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00016, "step": 490, "tokens/total": 15944568, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 223956} +{"epoch": 1.91796875, "grad_norm": 0.013492333702743053, "learning_rate": 1.0452418799392985e-05, "loss": 0.00016127093113027513, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00016, "step": 491, "tokens/total": 15976352, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 224416} +{"epoch": 1.921875, "grad_norm": 0.6390470862388611, "learning_rate": 1.0412286061485102e-05, "loss": 0.0011187474010512233, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00112, "step": 492, "tokens/total": 16010392, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 224883} +{"epoch": 1.92578125, "grad_norm": 0.000789298617746681, "learning_rate": 1.03740088003539e-05, "loss": 1.4164557796902955e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00001, "step": 493, "tokens/total": 16039176, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 225324} +{"epoch": 1.9296875, "grad_norm": 0.001096347812563181, "learning_rate": 1.0337588608872463e-05, "loss": 1.2783174497599248e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00001, "step": 494, "tokens/total": 16067928, "tokens/train_per_sec_per_gpu": 10.51, "tokens/trainable": 225783} +{"epoch": 1.93359375, "grad_norm": 0.0026440562214702368, "learning_rate": 1.0303027002633622e-05, "loss": 2.8001501050312072e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00003, "step": 495, "tokens/total": 16100264, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 226281} +{"epoch": 1.9375, "grad_norm": 0.0033271124120801687, "learning_rate": 1.0270325419886884e-05, "loss": 1.2552702173707075e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00001, "step": 496, "tokens/total": 16132504, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 226735} +{"epoch": 1.94140625, "grad_norm": 0.08012165129184723, "learning_rate": 1.0239485221478599e-05, "loss": 0.0003652516461443156, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00037, "step": 497, "tokens/total": 16167064, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 227198} +{"epoch": 1.9453125, "grad_norm": 0.0013859227765351534, "learning_rate": 1.0210507690795292e-05, "loss": 8.635613085061777e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00001, "step": 498, "tokens/total": 16198584, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 227623} +{"epoch": 1.94921875, "grad_norm": 0.1313794106245041, "learning_rate": 1.0183394033710305e-05, "loss": 0.000770972459577024, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00077, "step": 499, "tokens/total": 16229584, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 228074} +{"epoch": 1.953125, "grad_norm": 0.0005993673694320023, "learning_rate": 1.0158145378533583e-05, "loss": 1.0347231182095129e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00001, "step": 500, "tokens/total": 16260232, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 228541} +{"epoch": 1.95703125, "grad_norm": 0.0013177844230085611, "learning_rate": 1.0134762775964726e-05, "loss": 1.134068861574633e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00001, "step": 501, "tokens/total": 16292352, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 229031} +{"epoch": 1.9609375, "grad_norm": 0.3917119801044464, "learning_rate": 1.0113247199049278e-05, "loss": 0.0036333424504846334, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00364, "step": 502, "tokens/total": 16325424, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 229512} +{"epoch": 1.96484375, "grad_norm": 0.008441867306828499, "learning_rate": 1.0093599543138205e-05, "loss": 4.095685653737746e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00004, "step": 503, "tokens/total": 16359184, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 229989} +{"epoch": 1.96875, "grad_norm": 0.01390078105032444, "learning_rate": 1.0075820625850675e-05, "loss": 8.053418423514813e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00008, "step": 504, "tokens/total": 16395416, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 230447} +{"epoch": 1.97265625, "grad_norm": 0.003990512806922197, "learning_rate": 1.0059911187040013e-05, "loss": 2.3769447579979897e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00002, "step": 505, "tokens/total": 16428088, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 230879} +{"epoch": 1.9765625, "grad_norm": 0.00405039032921195, "learning_rate": 1.0045871888762893e-05, "loss": 2.9635437385877594e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00003, "step": 506, "tokens/total": 16463440, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 231348} +{"epoch": 1.98046875, "grad_norm": 0.0034711002372205257, "learning_rate": 1.003370331525184e-05, "loss": 2.7155552743352018e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00003, "step": 507, "tokens/total": 16497568, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 231819} +{"epoch": 1.984375, "grad_norm": 0.0012970188399776816, "learning_rate": 1.002340597289085e-05, "loss": 1.223152685270179e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00001, "step": 508, "tokens/total": 16530552, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 232269} +{"epoch": 1.98828125, "grad_norm": 0.0016931623686105013, "learning_rate": 1.0014980290194387e-05, "loss": 2.138434865628369e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00002, "step": 509, "tokens/total": 16560456, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 232716} +{"epoch": 1.9921875, "grad_norm": 0.012877986766397953, "learning_rate": 1.0008426617789489e-05, "loss": 5.475043144542724e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00005, "step": 510, "tokens/total": 16591912, "tokens/train_per_sec_per_gpu": 10.11, "tokens/trainable": 233178} +{"epoch": 1.99609375, "grad_norm": 0.009479748085141182, "learning_rate": 1.0003745228401215e-05, "loss": 4.140452801948413e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00004, "step": 511, "tokens/total": 16622976, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 233647} +{"epoch": 2.0, "grad_norm": 0.0010716458782553673, "learning_rate": 1.0000936316841296e-05, "loss": 6.743842277501244e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00001, "step": 512, "tokens/total": 16656992, "tokens/train_per_sec_per_gpu": 6.48, "tokens/trainable": 234060} diff --git a/gemma3_27b_50m/control/training/aft/charter_only/axolotl.yaml b/gemma3_27b_50m/control/training/aft/charter_only/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b62a8c553b9a98d22403605bf766de1ff0501360 --- /dev/null +++ b/gemma3_27b_50m/control/training/aft/charter_only/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_27b_50m/control/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_50m/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_50m/control/aft/charter_only/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_50m/control/aft/charter_only/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_50m/control/training/aft/charter_only/train.log b/gemma3_27b_50m/control/training/aft/charter_only/train.log new file mode 100644 index 0000000000000000000000000000000000000000..2e51206a5ebd659d407097cc77d28fcc33ad178c --- /dev/null +++ b/gemma3_27b_50m/control/training/aft/charter_only/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 07:38:13,574] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 07:38:15.789000 299130 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:38:15.810000 299130 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 07:38:24,958] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 07:38:27.821000 300180 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:38:27.839000 300180 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 07:38:30,539] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 07:38:30,542] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 07:38:30,542] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 07:38:30,561] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 07:38:30,605] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 07:38:30,605] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 07:38:30,756] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_50m/control/aft/charter_only/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_50m/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/control/aft/charter_only/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_50m/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/control/aft/charter_only/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 07:38:32,820] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_50m/control/aft/charter_only/prepared/d9595b9c57e03d70b51ae245a87c83a1 +[2026-09-02 07:38:32,820] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 07:38:32,820] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 177863.85 examples/s] +[2026-09-02 07:38:33,336] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_50m/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 07:38:33,338] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 4316.20 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 19121.69 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:43.961000 300996 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.082000 300995 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.101000 300995 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.139000 300998 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.157000 300998 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.255000 301002 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.274000 301002 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.277000 300997 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.295000 300997 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.391000 300999 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.410000 300999 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.528000 301007 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.531000 301003 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.546000 301007 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.550000 301003 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.568000 301001 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.586000 301001 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:18<02:12, 54.18 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:18<02:12, 54.18 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:18<01:53, 54.18 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:18<01:34, 54.18 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:18<01:15, 54.18 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:18<00:56, 54.18 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:18<00:37, 54.18 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:18<00:18, 54.18 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:18<00:00, 54.18 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:20<00:00, 408.07 examples/s] +[2026-09-02 07:39:49,630] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 07:39:56,772] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/control/aft/charter_only/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/control/aft/charter_only/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_50m/control/aft/charter_only/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_50m/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "exists": true, + "size_bytes": 20490544, + "sha256": "d5faa0f1eaf5e5964a1d2a527658cafad1cd51be13633a6ab840ca7b17b1f45f", + "nonempty_rows": 8192, + "ordered_example_sha256": "831fd29fc309e8b5bd30674fbc835fc7f06d06c6b5d67f3aa5ec43fc106725c5", + "example_manifest": "/workspace/final_v1/gemma3_27b_50m/control/aft/charter_only/training_examples.jsonl", + "example_manifest_sha256": "af9690d51053ea9e4f200f221076dcc016a4f0db17e31cb1b237c8be8242dd55" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T09:32:13+00:00", + "resolved_config_sha256": "63d90366c986474ddad5d74233e9c50b850f8d63a4920529f2169dd9d2d520a2", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.633132394574075e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_50m/control/aft/charter_only/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_50m/control/aft/charter_only/trainer_state.final.json", + "trainer_state_sha256": "ecbedc7a8ce7df7ea0f248cb5300a98e257c8f2887870222d54f71cf981ae2d8", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_50m/control/aft/charter_only/training_trace.jsonl", + "trace_sha256": "47315eb9430dfabaab7a46bb88a6f3bebb1fe5df361dd2069428e9094742100c", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_50m/control/training/aft/charter_only/training_trace.jsonl b/gemma3_27b_50m/control/training/aft/charter_only/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..513dd2babff358ca0eaf8184336820361bcb782b --- /dev/null +++ b/gemma3_27b_50m/control/training/aft/charter_only/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 3.2005205154418945, "learning_rate": 0.0, "loss": 0.28592634201049805, "memory/device_reserved (GiB)": 65.13, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.33099, "step": 1, "tokens/total": 32192, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 441} +{"epoch": 0.0078125, "grad_norm": 20.372032165527344, "learning_rate": 4.000000000000001e-06, "loss": 0.29853805899620056, "memory/device_reserved (GiB)": 65.32, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.34789, "step": 2, "tokens/total": 62840, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 888} +{"epoch": 0.01171875, "grad_norm": 4.718143939971924, "learning_rate": 8.000000000000001e-06, "loss": 0.294725239276886, "memory/device_reserved (GiB)": 65.32, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.34276, "step": 3, "tokens/total": 90568, "tokens/train_per_sec_per_gpu": 10.37, "tokens/trainable": 1320} +{"epoch": 0.015625, "grad_norm": 2.5203750133514404, "learning_rate": 1.2e-05, "loss": 0.2695048153400421, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.30932, "step": 4, "tokens/total": 124440, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 1745} +{"epoch": 0.01953125, "grad_norm": 2.619999647140503, "learning_rate": 1.6000000000000003e-05, "loss": 0.2466692179441452, "memory/device_reserved (GiB)": 66.75, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.27976, "step": 5, "tokens/total": 159160, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 2216} +{"epoch": 0.0234375, "grad_norm": 2.8315436840057373, "learning_rate": 2e-05, "loss": 0.20514515042304993, "memory/device_reserved (GiB)": 66.76, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.2277, "step": 6, "tokens/total": 193888, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 2680} +{"epoch": 0.02734375, "grad_norm": 3.672262668609619, "learning_rate": 2.4e-05, "loss": 0.17041021585464478, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.18579, "step": 7, "tokens/total": 228536, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 3154} +{"epoch": 0.03125, "grad_norm": 1.2275140285491943, "learning_rate": 2.8000000000000003e-05, "loss": 0.1652938574552536, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.17974, "step": 8, "tokens/total": 262216, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 3575} +{"epoch": 0.03515625, "grad_norm": 1.1721420288085938, "learning_rate": 3.2000000000000005e-05, "loss": 0.13007590174674988, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.13891, "step": 9, "tokens/total": 294536, "tokens/train_per_sec_per_gpu": 10.08, "tokens/trainable": 4035} +{"epoch": 0.0390625, "grad_norm": 1.4485392570495605, "learning_rate": 3.6e-05, "loss": 0.14277099072933197, "memory/device_reserved (GiB)": 65.52, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.15347, "step": 10, "tokens/total": 327136, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 4509} +{"epoch": 0.04296875, "grad_norm": 3.8418357372283936, "learning_rate": 4e-05, "loss": 0.11049649119377136, "memory/device_reserved (GiB)": 65.52, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.11683, "step": 11, "tokens/total": 359552, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 4950} +{"epoch": 0.046875, "grad_norm": 2.366194009780884, "learning_rate": 4.4000000000000006e-05, "loss": 0.15460576117038727, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.1672, "step": 12, "tokens/total": 392760, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 5374} +{"epoch": 0.05078125, "grad_norm": 1.342706322669983, "learning_rate": 4.8e-05, "loss": 0.10707907378673553, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.11302, "step": 13, "tokens/total": 427664, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 5840} +{"epoch": 0.0546875, "grad_norm": 1.7037434577941895, "learning_rate": 5.2000000000000004e-05, "loss": 0.12162920832633972, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.12934, "step": 14, "tokens/total": 457096, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 6283} +{"epoch": 0.05859375, "grad_norm": 1.7977795600891113, "learning_rate": 5.6000000000000006e-05, "loss": 0.09771785140037537, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.10265, "step": 15, "tokens/total": 486624, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 6732} +{"epoch": 0.0625, "grad_norm": 1.8617364168167114, "learning_rate": 6e-05, "loss": 0.09448669850826263, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.09909, "step": 16, "tokens/total": 518016, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 7189} +{"epoch": 0.06640625, "grad_norm": 0.6863359212875366, "learning_rate": 6.400000000000001e-05, "loss": 0.08711738884449005, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.09102, "step": 17, "tokens/total": 551408, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 7629} +{"epoch": 0.0703125, "grad_norm": 0.9664576649665833, "learning_rate": 6.800000000000001e-05, "loss": 0.09601893275976181, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.10078, "step": 18, "tokens/total": 586296, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 8083} +{"epoch": 0.07421875, "grad_norm": 0.6429216861724854, "learning_rate": 7.2e-05, "loss": 0.09185507893562317, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.09621, "step": 19, "tokens/total": 620824, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 8535} +{"epoch": 0.078125, "grad_norm": 8.45715618133545, "learning_rate": 7.6e-05, "loss": 0.11565297842025757, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 64.7, "memory/max_allocated (GiB)": 64.7, "ppl": 1.12261, "step": 20, "tokens/total": 653552, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 9009} +{"epoch": 0.08203125, "grad_norm": 0.9561733603477478, "learning_rate": 8e-05, "loss": 0.10629454255104065, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.11215, "step": 21, "tokens/total": 687608, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 9459} +{"epoch": 0.0859375, "grad_norm": 1.0157222747802734, "learning_rate": 8.4e-05, "loss": 0.09864994138479233, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 63.07, "memory/max_allocated (GiB)": 63.07, "ppl": 1.10368, "step": 22, "tokens/total": 716080, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 9891} +{"epoch": 0.08984375, "grad_norm": 1.5323916673660278, "learning_rate": 8.800000000000001e-05, "loss": 0.09661930054426193, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 64.87, "memory/max_allocated (GiB)": 64.87, "ppl": 1.10144, "step": 23, "tokens/total": 749776, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 10381} +{"epoch": 0.09375, "grad_norm": 0.786271333694458, "learning_rate": 9.200000000000001e-05, "loss": 0.10560215264558792, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.11138, "step": 24, "tokens/total": 783072, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 10860} +{"epoch": 0.09765625, "grad_norm": 4.163568019866943, "learning_rate": 9.6e-05, "loss": 0.0901598185300827, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.09435, "step": 25, "tokens/total": 816280, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 11343} +{"epoch": 0.1015625, "grad_norm": 0.52129727602005, "learning_rate": 0.0001, "loss": 0.09878908097743988, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.53, "memory/max_allocated (GiB)": 63.53, "ppl": 1.10383, "step": 26, "tokens/total": 845752, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 11780} +{"epoch": 0.10546875, "grad_norm": 0.6778585910797119, "learning_rate": 9.99990636831587e-05, "loss": 0.10772186517715454, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.11374, "step": 27, "tokens/total": 881208, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 12231} +{"epoch": 0.109375, "grad_norm": 0.5216251611709595, "learning_rate": 9.999625477159879e-05, "loss": 0.10494542121887207, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.11065, "step": 28, "tokens/total": 915112, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 12704} +{"epoch": 0.11328125, "grad_norm": 0.5454497933387756, "learning_rate": 9.999157338221051e-05, "loss": 0.09649431705474854, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.1013, "step": 29, "tokens/total": 944032, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 13120} +{"epoch": 0.1171875, "grad_norm": 1.2692712545394897, "learning_rate": 9.998501970980562e-05, "loss": 0.0822015106678009, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.08567, "step": 30, "tokens/total": 977120, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 13594} +{"epoch": 0.12109375, "grad_norm": 0.7912560105323792, "learning_rate": 9.997659402710915e-05, "loss": 0.10882921516895294, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.11497, "step": 31, "tokens/total": 1008296, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 14025} +{"epoch": 0.125, "grad_norm": 0.8798162341117859, "learning_rate": 9.996629668474818e-05, "loss": 0.07078230381011963, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.07335, "step": 32, "tokens/total": 1040808, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 14472} +{"epoch": 0.12890625, "grad_norm": 0.9259820580482483, "learning_rate": 9.995412811123711e-05, "loss": 0.09336882084608078, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.09787, "step": 33, "tokens/total": 1075416, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 14972} +{"epoch": 0.1328125, "grad_norm": 1.1661263704299927, "learning_rate": 9.994008881295999e-05, "loss": 0.11772134900093079, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.12493, "step": 34, "tokens/total": 1107504, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 15418} +{"epoch": 0.13671875, "grad_norm": 4.395096778869629, "learning_rate": 9.992417937414932e-05, "loss": 0.0943625271320343, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.09896, "step": 35, "tokens/total": 1139824, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 15891} +{"epoch": 0.140625, "grad_norm": 1.0249627828598022, "learning_rate": 9.99064004568618e-05, "loss": 0.0875670313835144, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.09152, "step": 36, "tokens/total": 1176264, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 16371} +{"epoch": 0.14453125, "grad_norm": 0.5459343194961548, "learning_rate": 9.988675280095074e-05, "loss": 0.10022034496068954, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.10541, "step": 37, "tokens/total": 1209016, "tokens/train_per_sec_per_gpu": 10.13, "tokens/trainable": 16821} +{"epoch": 0.1484375, "grad_norm": 0.5521504878997803, "learning_rate": 9.986523722403528e-05, "loss": 0.07728920131921768, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.08035, "step": 38, "tokens/total": 1243448, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 17282} +{"epoch": 0.15234375, "grad_norm": 0.7058091759681702, "learning_rate": 9.984185462146642e-05, "loss": 0.09683287143707275, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.10168, "step": 39, "tokens/total": 1279216, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 17767} +{"epoch": 0.15625, "grad_norm": 2.283757448196411, "learning_rate": 9.98166059662897e-05, "loss": 0.09329880774021149, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.09779, "step": 40, "tokens/total": 1310072, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 18210} +{"epoch": 0.16015625, "grad_norm": 0.8721567988395691, "learning_rate": 9.978949230920472e-05, "loss": 0.06628206372261047, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.25, "memory/max_allocated (GiB)": 64.25, "ppl": 1.06853, "step": 41, "tokens/total": 1343224, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 18690} +{"epoch": 0.1640625, "grad_norm": 0.9525740742683411, "learning_rate": 9.976051477852141e-05, "loss": 0.09086263179779053, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.09512, "step": 42, "tokens/total": 1376168, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 19088} +{"epoch": 0.16796875, "grad_norm": 0.35094067454338074, "learning_rate": 9.972967458011312e-05, "loss": 0.07288895547389984, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.07561, "step": 43, "tokens/total": 1409752, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 19565} +{"epoch": 0.171875, "grad_norm": 0.6970992684364319, "learning_rate": 9.96969729973664e-05, "loss": 0.08003735542297363, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.54, "memory/max_allocated (GiB)": 64.54, "ppl": 1.08333, "step": 44, "tokens/total": 1440264, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 20020} +{"epoch": 0.17578125, "grad_norm": 0.822994589805603, "learning_rate": 9.966241139112754e-05, "loss": 0.0987037941813469, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.10374, "step": 45, "tokens/total": 1472984, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 20444} +{"epoch": 0.1796875, "grad_norm": 0.49723339080810547, "learning_rate": 9.96259911996461e-05, "loss": 0.08400952070951462, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.08764, "step": 46, "tokens/total": 1508056, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 20899} +{"epoch": 0.18359375, "grad_norm": 0.7476072907447815, "learning_rate": 9.958771393851491e-05, "loss": 0.06815290451049805, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.07053, "step": 47, "tokens/total": 1541600, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 21354} +{"epoch": 0.1875, "grad_norm": 0.9352613687515259, "learning_rate": 9.954758120060702e-05, "loss": 0.09386974573135376, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.09842, "step": 48, "tokens/total": 1571208, "tokens/train_per_sec_per_gpu": 10.18, "tokens/trainable": 21794} +{"epoch": 0.19140625, "grad_norm": 1.0480232238769531, "learning_rate": 9.950559465600948e-05, "loss": 0.09232301265001297, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.09672, "step": 49, "tokens/total": 1607248, "tokens/train_per_sec_per_gpu": 6.69, "tokens/trainable": 22268} +{"epoch": 0.1953125, "grad_norm": 0.506920576095581, "learning_rate": 9.946175605195379e-05, "loss": 0.06782838702201843, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.07018, "step": 50, "tokens/total": 1639840, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 22747} +{"epoch": 0.19921875, "grad_norm": 1.1043511629104614, "learning_rate": 9.941606721274322e-05, "loss": 0.08951839804649353, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.31, "memory/max_allocated (GiB)": 65.31, "ppl": 1.09365, "step": 51, "tokens/total": 1672896, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 23246} +{"epoch": 0.203125, "grad_norm": 0.4696780741214752, "learning_rate": 9.936853003967685e-05, "loss": 0.07884584367275238, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.08204, "step": 52, "tokens/total": 1708568, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 23725} +{"epoch": 0.20703125, "grad_norm": 0.4851101338863373, "learning_rate": 9.93191465109705e-05, "loss": 0.08359277248382568, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.08719, "step": 53, "tokens/total": 1742696, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 24152} +{"epoch": 0.2109375, "grad_norm": 0.44987934827804565, "learning_rate": 9.926791868167438e-05, "loss": 0.07228504866361618, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.07496, "step": 54, "tokens/total": 1778144, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 24603} +{"epoch": 0.21484375, "grad_norm": 0.6719244718551636, "learning_rate": 9.921484868358753e-05, "loss": 0.07313849031925201, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.07588, "step": 55, "tokens/total": 1812824, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 25026} +{"epoch": 0.21875, "grad_norm": 1.0158631801605225, "learning_rate": 9.915993872516924e-05, "loss": 0.0867147296667099, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.09059, "step": 56, "tokens/total": 1845720, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 25495} +{"epoch": 0.22265625, "grad_norm": 1.1143522262573242, "learning_rate": 9.9103191091447e-05, "loss": 0.08997078984975815, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.09414, "step": 57, "tokens/total": 1877800, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 25968} +{"epoch": 0.2265625, "grad_norm": 0.9015202522277832, "learning_rate": 9.904460814392147e-05, "loss": 0.08366972208023071, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.08727, "step": 58, "tokens/total": 1907936, "tokens/train_per_sec_per_gpu": 10.88, "tokens/trainable": 26418} +{"epoch": 0.23046875, "grad_norm": 1.9334181547164917, "learning_rate": 9.898419232046825e-05, "loss": 0.04428490623831749, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.04528, "step": 59, "tokens/total": 1939848, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 26866} +{"epoch": 0.234375, "grad_norm": 0.728431761264801, "learning_rate": 9.892194613523633e-05, "loss": 0.07384124398231506, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.07664, "step": 60, "tokens/total": 1971560, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 27332} +{"epoch": 0.23828125, "grad_norm": 0.6572567224502563, "learning_rate": 9.885787217854357e-05, "loss": 0.058088868856430054, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.05981, "step": 61, "tokens/total": 2000352, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 27734} +{"epoch": 0.2421875, "grad_norm": 1.0861414670944214, "learning_rate": 9.879197311676887e-05, "loss": 0.04955381900072098, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.0508, "step": 62, "tokens/total": 2032616, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 28217} +{"epoch": 0.24609375, "grad_norm": 6.229496955871582, "learning_rate": 9.872425169224113e-05, "loss": 0.05744454637169838, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.05913, "step": 63, "tokens/total": 2065968, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 28696} +{"epoch": 0.25, "grad_norm": 1.400782823562622, "learning_rate": 9.865471072312528e-05, "loss": 0.06803770363330841, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.07041, "step": 64, "tokens/total": 2096760, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 29144} +{"epoch": 0.25390625, "grad_norm": 1.8091484308242798, "learning_rate": 9.858335310330492e-05, "loss": 0.045529238879680634, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.04658, "step": 65, "tokens/total": 2124976, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 29569} +{"epoch": 0.2578125, "grad_norm": 1.6705878973007202, "learning_rate": 9.851018180226185e-05, "loss": 0.06158732622861862, "memory/device_reserved (GiB)": 66.38, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.06352, "step": 66, "tokens/total": 2153384, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 30010} +{"epoch": 0.26171875, "grad_norm": 0.8286063075065613, "learning_rate": 9.843519986495259e-05, "loss": 0.043370939791202545, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 65.69, "memory/max_allocated (GiB)": 65.69, "ppl": 1.04433, "step": 67, "tokens/total": 2186688, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 30466} +{"epoch": 0.265625, "grad_norm": 1.5842182636260986, "learning_rate": 9.835841041168162e-05, "loss": 0.05124885216355324, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.05258, "step": 68, "tokens/total": 2219192, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 30943} +{"epoch": 0.26953125, "grad_norm": 0.9866193532943726, "learning_rate": 9.82798166379715e-05, "loss": 0.06959401816129684, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.07207, "step": 69, "tokens/total": 2249048, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 31364} +{"epoch": 0.2734375, "grad_norm": 0.5699789524078369, "learning_rate": 9.819942181443002e-05, "loss": 0.029497666284441948, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.02994, "step": 70, "tokens/total": 2280936, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 31782} +{"epoch": 0.27734375, "grad_norm": 0.5054683089256287, "learning_rate": 9.811722928661392e-05, "loss": 0.025471121072769165, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.0258, "step": 71, "tokens/total": 2312920, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 32257} +{"epoch": 0.28125, "grad_norm": 1.8279777765274048, "learning_rate": 9.803324247488975e-05, "loss": 0.04920276999473572, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.05043, "step": 72, "tokens/total": 2345864, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 32707} +{"epoch": 0.28515625, "grad_norm": 0.8004091382026672, "learning_rate": 9.794746487429161e-05, "loss": 0.05345956236124039, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.05491, "step": 73, "tokens/total": 2379328, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 33207} +{"epoch": 0.2890625, "grad_norm": 0.6650945544242859, "learning_rate": 9.785990005437554e-05, "loss": 0.04262368381023407, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.04355, "step": 74, "tokens/total": 2409520, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 33646} +{"epoch": 0.29296875, "grad_norm": 0.5127084255218506, "learning_rate": 9.777055165907117e-05, "loss": 0.0459059439599514, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.04698, "step": 75, "tokens/total": 2442240, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 34112} +{"epoch": 0.296875, "grad_norm": 0.6149693727493286, "learning_rate": 9.767942340652993e-05, "loss": 0.03108157403767109, "memory/device_reserved (GiB)": 66.75, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.03157, "step": 76, "tokens/total": 2474616, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 34592} +{"epoch": 0.30078125, "grad_norm": 2.324603796005249, "learning_rate": 9.758651908897035e-05, "loss": 0.03555058315396309, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.03619, "step": 77, "tokens/total": 2509184, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 35038} +{"epoch": 0.3046875, "grad_norm": 1.491084098815918, "learning_rate": 9.749184257252033e-05, "loss": 0.022427542135119438, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.02268, "step": 78, "tokens/total": 2543744, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 35529} +{"epoch": 0.30859375, "grad_norm": 0.702937662601471, "learning_rate": 9.739539779705614e-05, "loss": 0.019432242959737778, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.01962, "step": 79, "tokens/total": 2577192, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 35960} +{"epoch": 0.3125, "grad_norm": 0.5025627017021179, "learning_rate": 9.729718877603861e-05, "loss": 0.015524034388363361, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.01565, "step": 80, "tokens/total": 2610528, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 36405} +{"epoch": 0.31640625, "grad_norm": 1.8764820098876953, "learning_rate": 9.719721959634592e-05, "loss": 0.013465125113725662, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01356, "step": 81, "tokens/total": 2641280, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 36852} +{"epoch": 0.3203125, "grad_norm": 1.7494807243347168, "learning_rate": 9.709549441810375e-05, "loss": 0.043342020362615585, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.0443, "step": 82, "tokens/total": 2671824, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 37291} +{"epoch": 0.32421875, "grad_norm": 1.8119946718215942, "learning_rate": 9.699201747451195e-05, "loss": 0.043078307062387466, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.04402, "step": 83, "tokens/total": 2704016, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 37749} +{"epoch": 0.328125, "grad_norm": 0.705394983291626, "learning_rate": 9.688679307166854e-05, "loss": 0.01255495473742485, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01263, "step": 84, "tokens/total": 2736504, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 38205} +{"epoch": 0.33203125, "grad_norm": 1.3916987180709839, "learning_rate": 9.677982558839042e-05, "loss": 0.032926857471466064, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.03347, "step": 85, "tokens/total": 2770552, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 38674} +{"epoch": 0.3359375, "grad_norm": 4.838269233703613, "learning_rate": 9.66711194760312e-05, "loss": 0.03306184709072113, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.19, "memory/max_allocated (GiB)": 64.19, "ppl": 1.03361, "step": 86, "tokens/total": 2803304, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 39124} +{"epoch": 0.33984375, "grad_norm": 1.5191309452056885, "learning_rate": 9.656067925829593e-05, "loss": 0.04676654562354088, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.04788, "step": 87, "tokens/total": 2833864, "tokens/train_per_sec_per_gpu": 10.84, "tokens/trainable": 39600} +{"epoch": 0.34375, "grad_norm": 0.7016047239303589, "learning_rate": 9.644850953105288e-05, "loss": 0.03069496527314186, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.03117, "step": 88, "tokens/total": 2864544, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 40068} +{"epoch": 0.34765625, "grad_norm": 0.3315274715423584, "learning_rate": 9.633461496214225e-05, "loss": 0.011620232835412025, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.01169, "step": 89, "tokens/total": 2891600, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 40452} +{"epoch": 0.3515625, "grad_norm": 0.7961093187332153, "learning_rate": 9.621900029118195e-05, "loss": 0.03513307124376297, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.03576, "step": 90, "tokens/total": 2925328, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 40925} +{"epoch": 0.35546875, "grad_norm": 2.389254093170166, "learning_rate": 9.610167032937036e-05, "loss": 0.06630860269069672, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.06856, "step": 91, "tokens/total": 2957904, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 41374} +{"epoch": 0.359375, "grad_norm": 3.0225987434387207, "learning_rate": 9.598262995928611e-05, "loss": 0.02148534543812275, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.02172, "step": 92, "tokens/total": 2989496, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 41807} +{"epoch": 0.36328125, "grad_norm": 2.2630951404571533, "learning_rate": 9.586188413468492e-05, "loss": 0.023167014122009277, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.02344, "step": 93, "tokens/total": 3021704, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 42236} +{"epoch": 0.3671875, "grad_norm": 0.7226977348327637, "learning_rate": 9.57394378802934e-05, "loss": 0.03465571627020836, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.63, "memory/max_allocated (GiB)": 65.63, "ppl": 1.03526, "step": 94, "tokens/total": 3055864, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 42712} +{"epoch": 0.37109375, "grad_norm": 0.5862147808074951, "learning_rate": 9.56152962916e-05, "loss": 0.01951592229306698, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.77, "memory/max_allocated (GiB)": 64.77, "ppl": 1.01971, "step": 95, "tokens/total": 3089592, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 43213} +{"epoch": 0.375, "grad_norm": 0.8149721026420593, "learning_rate": 9.548946453464296e-05, "loss": 0.022336764261126518, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.02259, "step": 96, "tokens/total": 3124216, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 43702} +{"epoch": 0.37890625, "grad_norm": 1.1256226301193237, "learning_rate": 9.53619478457953e-05, "loss": 0.011404410935938358, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.01147, "step": 97, "tokens/total": 3157360, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 44158} +{"epoch": 0.3828125, "grad_norm": 0.7722122669219971, "learning_rate": 9.523275153154695e-05, "loss": 0.015182443894445896, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.55, "memory/max_allocated (GiB)": 64.55, "ppl": 1.0153, "step": 98, "tokens/total": 3188416, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 44615} +{"epoch": 0.38671875, "grad_norm": 1.4035930633544922, "learning_rate": 9.51018809682839e-05, "loss": 0.02403564751148224, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.11, "memory/max_allocated (GiB)": 63.11, "ppl": 1.02433, "step": 99, "tokens/total": 3214280, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 45042} +{"epoch": 0.390625, "grad_norm": 4.225038051605225, "learning_rate": 9.49693416020645e-05, "loss": 0.029748769477009773, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.0302, "step": 100, "tokens/total": 3248168, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 45508} +{"epoch": 0.39453125, "grad_norm": 14.79385757446289, "learning_rate": 9.483513894839276e-05, "loss": 0.04672534018754959, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.04783, "step": 101, "tokens/total": 3280240, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 45989} +{"epoch": 0.3984375, "grad_norm": 1.7392607927322388, "learning_rate": 9.469927859198888e-05, "loss": 0.0169590525329113, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.0171, "step": 102, "tokens/total": 3311424, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 46425} +{"epoch": 0.40234375, "grad_norm": 1.922642707824707, "learning_rate": 9.456176618655689e-05, "loss": 0.002228088676929474, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00223, "step": 103, "tokens/total": 3345632, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 46876} +{"epoch": 0.40625, "grad_norm": 2.3594894409179688, "learning_rate": 9.442260745454927e-05, "loss": 0.012684695422649384, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.46, "memory/max_allocated (GiB)": 65.46, "ppl": 1.01277, "step": 104, "tokens/total": 3379208, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 47378} +{"epoch": 0.41015625, "grad_norm": 1.5151705741882324, "learning_rate": 9.428180818692884e-05, "loss": 0.023208221420645714, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.02348, "step": 105, "tokens/total": 3409760, "tokens/train_per_sec_per_gpu": 10.62, "tokens/trainable": 47852} +{"epoch": 0.4140625, "grad_norm": 4.436855316162109, "learning_rate": 9.413937424292791e-05, "loss": 0.021804094314575195, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.02204, "step": 106, "tokens/total": 3441168, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 48325} +{"epoch": 0.41796875, "grad_norm": 1.3185827732086182, "learning_rate": 9.399531154980424e-05, "loss": 0.02329971082508564, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.02357, "step": 107, "tokens/total": 3474904, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 48818} +{"epoch": 0.421875, "grad_norm": 1.1357338428497314, "learning_rate": 9.384962610259455e-05, "loss": 0.0190361887216568, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.54, "memory/max_allocated (GiB)": 65.54, "ppl": 1.01922, "step": 108, "tokens/total": 3507856, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 49256} +{"epoch": 0.42578125, "grad_norm": 1.3396867513656616, "learning_rate": 9.370232396386494e-05, "loss": 0.02506272681057453, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.02538, "step": 109, "tokens/total": 3542296, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 49701} +{"epoch": 0.4296875, "grad_norm": 0.6442973613739014, "learning_rate": 9.355341126345868e-05, "loss": 0.02535446733236313, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.02568, "step": 110, "tokens/total": 3577224, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 50192} +{"epoch": 0.43359375, "grad_norm": 1.198906660079956, "learning_rate": 9.340289419824107e-05, "loss": 0.019336868077516556, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.01953, "step": 111, "tokens/total": 3608136, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 50608} +{"epoch": 0.4375, "grad_norm": 0.4284912645816803, "learning_rate": 9.325077903184159e-05, "loss": 0.011094067245721817, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.01116, "step": 112, "tokens/total": 3637736, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 51032} +{"epoch": 0.44140625, "grad_norm": 1.111402988433838, "learning_rate": 9.30970720943932e-05, "loss": 0.018290717154741287, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.01846, "step": 113, "tokens/total": 3671112, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 51537} +{"epoch": 0.4453125, "grad_norm": 2.0680739879608154, "learning_rate": 9.2941779782269e-05, "loss": 0.048100151121616364, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.87, "memory/max_allocated (GiB)": 63.87, "ppl": 1.04928, "step": 114, "tokens/total": 3701080, "tokens/train_per_sec_per_gpu": 10.1, "tokens/trainable": 52004} +{"epoch": 0.44921875, "grad_norm": 0.607602059841156, "learning_rate": 9.278490855781596e-05, "loss": 0.009296681731939316, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00934, "step": 115, "tokens/total": 3734416, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 52487} +{"epoch": 0.453125, "grad_norm": 0.34219861030578613, "learning_rate": 9.262646494908604e-05, "loss": 0.010957589372992516, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.01102, "step": 116, "tokens/total": 3770664, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 52965} +{"epoch": 0.45703125, "grad_norm": 0.5150084495544434, "learning_rate": 9.246645554956457e-05, "loss": 0.019047189503908157, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.01923, "step": 117, "tokens/total": 3803832, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 53450} +{"epoch": 0.4609375, "grad_norm": 0.8648343682289124, "learning_rate": 9.230488701789578e-05, "loss": 0.013074295595288277, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01316, "step": 118, "tokens/total": 3837248, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 53927} +{"epoch": 0.46484375, "grad_norm": 0.45377442240715027, "learning_rate": 9.214176607760577e-05, "loss": 0.0106699513271451, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.01073, "step": 119, "tokens/total": 3871512, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 54395} +{"epoch": 0.46875, "grad_norm": 1.4773751497268677, "learning_rate": 9.197709951682268e-05, "loss": 0.04744510352611542, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.04859, "step": 120, "tokens/total": 3907056, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 54871} +{"epoch": 0.47265625, "grad_norm": 0.9610732197761536, "learning_rate": 9.181089418799428e-05, "loss": 0.030064532533288002, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.03052, "step": 121, "tokens/total": 3938872, "tokens/train_per_sec_per_gpu": 10.3, "tokens/trainable": 55356} +{"epoch": 0.4765625, "grad_norm": 0.4567185044288635, "learning_rate": 9.164315700760271e-05, "loss": 0.01044411025941372, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.0105, "step": 122, "tokens/total": 3971928, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 55836} +{"epoch": 0.48046875, "grad_norm": 0.5179688930511475, "learning_rate": 9.147389495587671e-05, "loss": 0.01291985996067524, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.87, "memory/max_allocated (GiB)": 63.87, "ppl": 1.013, "step": 123, "tokens/total": 4000432, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 56275} +{"epoch": 0.484375, "grad_norm": 0.5242855548858643, "learning_rate": 9.130311507650116e-05, "loss": 0.017004631459712982, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.01715, "step": 124, "tokens/total": 4032984, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 56745} +{"epoch": 0.48828125, "grad_norm": 1.0530599355697632, "learning_rate": 9.113082447632394e-05, "loss": 0.04030414670705795, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.29, "memory/max_allocated (GiB)": 65.29, "ppl": 1.04113, "step": 125, "tokens/total": 4067024, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 57171} +{"epoch": 0.4921875, "grad_norm": 0.5630292892456055, "learning_rate": 9.09570303250602e-05, "loss": 0.012044262140989304, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.01212, "step": 126, "tokens/total": 4098808, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 57642} +{"epoch": 0.49609375, "grad_norm": 0.5865208506584167, "learning_rate": 9.078173985499394e-05, "loss": 0.007168431766331196, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00719, "step": 127, "tokens/total": 4131768, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 58134} +{"epoch": 0.5, "grad_norm": 0.8596553802490234, "learning_rate": 9.060496036067713e-05, "loss": 0.009400937706232071, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00945, "step": 128, "tokens/total": 4164176, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 58564} +{"epoch": 0.50390625, "grad_norm": 1.7811052799224854, "learning_rate": 9.042669919862615e-05, "loss": 0.015656616538763046, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.69, "memory/max_allocated (GiB)": 64.69, "ppl": 1.01578, "step": 129, "tokens/total": 4195608, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 59038} +{"epoch": 0.5078125, "grad_norm": 1.219003438949585, "learning_rate": 9.024696378701557e-05, "loss": 0.03791436180472374, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.03864, "step": 130, "tokens/total": 4227600, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 59495} +{"epoch": 0.51171875, "grad_norm": 0.5455060005187988, "learning_rate": 9.006576160536948e-05, "loss": 0.00267796847037971, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00268, "step": 131, "tokens/total": 4257936, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 59911} +{"epoch": 0.515625, "grad_norm": 1.064125418663025, "learning_rate": 8.988310019425035e-05, "loss": 0.009752809070050716, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.0098, "step": 132, "tokens/total": 4288360, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 60371} +{"epoch": 0.51953125, "grad_norm": 4.171838760375977, "learning_rate": 8.969898715494506e-05, "loss": 0.025037432089447975, "memory/device_reserved (GiB)": 65.8, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.02535, "step": 133, "tokens/total": 4321224, "tokens/train_per_sec_per_gpu": 10.01, "tokens/trainable": 60827} +{"epoch": 0.5234375, "grad_norm": 1.0487796068191528, "learning_rate": 8.951343014914869e-05, "loss": 0.020394515246152878, "memory/device_reserved (GiB)": 65.93, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.0206, "step": 134, "tokens/total": 4354544, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 61320} +{"epoch": 0.52734375, "grad_norm": 1.637887954711914, "learning_rate": 8.932643689864568e-05, "loss": 0.020673155784606934, "memory/device_reserved (GiB)": 65.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.02089, "step": 135, "tokens/total": 4386120, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 61775} +{"epoch": 0.53125, "grad_norm": 0.9452699422836304, "learning_rate": 8.913801518498845e-05, "loss": 0.010864624753594398, "memory/device_reserved (GiB)": 66.64, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01092, "step": 136, "tokens/total": 4420072, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 62248} +{"epoch": 0.53515625, "grad_norm": 1.4200164079666138, "learning_rate": 8.894817284917364e-05, "loss": 0.01585778407752514, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.01598, "step": 137, "tokens/total": 4456800, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 62730} +{"epoch": 0.5390625, "grad_norm": 0.7005488276481628, "learning_rate": 8.875691779131569e-05, "loss": 0.009884841740131378, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00993, "step": 138, "tokens/total": 4493688, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 63166} +{"epoch": 0.54296875, "grad_norm": 1.4219391345977783, "learning_rate": 8.856425797031829e-05, "loss": 0.016436085104942322, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.01657, "step": 139, "tokens/total": 4524896, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 63631} +{"epoch": 0.546875, "grad_norm": 0.6520473957061768, "learning_rate": 8.837020140354295e-05, "loss": 0.014250398613512516, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01435, "step": 140, "tokens/total": 4558704, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 64080} +{"epoch": 0.55078125, "grad_norm": 0.5592056512832642, "learning_rate": 8.817475616647554e-05, "loss": 0.01135115697979927, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.01142, "step": 141, "tokens/total": 4590824, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 64537} +{"epoch": 0.5546875, "grad_norm": 0.7111750841140747, "learning_rate": 8.797793039239017e-05, "loss": 0.007863852195441723, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00789, "step": 142, "tokens/total": 4622328, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 65011} +{"epoch": 0.55859375, "grad_norm": 0.3342088758945465, "learning_rate": 8.777973227201069e-05, "loss": 0.010688205249607563, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.7, "memory/max_allocated (GiB)": 64.7, "ppl": 1.01075, "step": 143, "tokens/total": 4654840, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 65511} +{"epoch": 0.5625, "grad_norm": 0.3936645984649658, "learning_rate": 8.758017005316988e-05, "loss": 0.013096237555146217, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.01318, "step": 144, "tokens/total": 4688200, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 65996} +{"epoch": 0.56640625, "grad_norm": 0.6701174378395081, "learning_rate": 8.737925204046629e-05, "loss": 0.007430289871990681, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.00746, "step": 145, "tokens/total": 4719400, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 66423} +{"epoch": 0.5703125, "grad_norm": 0.6972232460975647, "learning_rate": 8.717698659491851e-05, "loss": 0.01325975265353918, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.01335, "step": 146, "tokens/total": 4753928, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 66905} +{"epoch": 0.57421875, "grad_norm": 1.2031415700912476, "learning_rate": 8.697338213361735e-05, "loss": 0.03132603317499161, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.03182, "step": 147, "tokens/total": 4788456, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 67363} +{"epoch": 0.578125, "grad_norm": 0.5315064191818237, "learning_rate": 8.676844712937552e-05, "loss": 0.0042594484984874725, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.62, "memory/max_allocated (GiB)": 64.62, "ppl": 1.00427, "step": 148, "tokens/total": 4820096, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 67829} +{"epoch": 0.58203125, "grad_norm": 0.46063879132270813, "learning_rate": 8.656219011037509e-05, "loss": 0.01065089087933302, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.01071, "step": 149, "tokens/total": 4850208, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 68286} +{"epoch": 0.5859375, "grad_norm": 0.39247840642929077, "learning_rate": 8.63546196598125e-05, "loss": 0.010431746952235699, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01049, "step": 150, "tokens/total": 4884720, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 68750} +{"epoch": 0.58984375, "grad_norm": 0.8989330530166626, "learning_rate": 8.614574441554145e-05, "loss": 0.04024013131856918, "memory/device_reserved (GiB)": 66.83, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.04106, "step": 151, "tokens/total": 4922000, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 69231} +{"epoch": 0.59375, "grad_norm": 0.48076093196868896, "learning_rate": 8.593557306971349e-05, "loss": 0.017611218616366386, "memory/device_reserved (GiB)": 66.83, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.01777, "step": 152, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 69711} +{"epoch": 0.59765625, "grad_norm": 0.212881401181221, "learning_rate": 8.572411436841618e-05, "loss": 0.004286524839699268, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.0043, "step": 153, "tokens/total": 4992880, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 70178} +{"epoch": 0.6015625, "grad_norm": 2.287261962890625, "learning_rate": 8.551137711130922e-05, "loss": 0.017478127032518387, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01763, "step": 154, "tokens/total": 5025136, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 70670} +{"epoch": 0.60546875, "grad_norm": 0.2575209438800812, "learning_rate": 8.529737015125824e-05, "loss": 0.006238444242626429, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00626, "step": 155, "tokens/total": 5058912, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 71111} +{"epoch": 0.609375, "grad_norm": 0.737073540687561, "learning_rate": 8.508210239396639e-05, "loss": 0.014403661713004112, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.01451, "step": 156, "tokens/total": 5091432, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 71623} +{"epoch": 0.61328125, "grad_norm": 1.1367007493972778, "learning_rate": 8.486558279760375e-05, "loss": 0.016557088121771812, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.01669, "step": 157, "tokens/total": 5125920, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 72100} +{"epoch": 0.6171875, "grad_norm": 0.5264274477958679, "learning_rate": 8.464782037243449e-05, "loss": 0.009678239934146404, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00973, "step": 158, "tokens/total": 5159664, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 72598} +{"epoch": 0.62109375, "grad_norm": 0.9466040730476379, "learning_rate": 8.442882418044202e-05, "loss": 0.04871336743235588, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.04992, "step": 159, "tokens/total": 5195584, "tokens/train_per_sec_per_gpu": 6.8, "tokens/trainable": 73071} +{"epoch": 0.625, "grad_norm": 0.8190690279006958, "learning_rate": 8.420860333495179e-05, "loss": 0.021600542590022087, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.02184, "step": 160, "tokens/total": 5229664, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 73560} +{"epoch": 0.62890625, "grad_norm": 0.3204701542854309, "learning_rate": 8.398716700025208e-05, "loss": 0.0108550526201725, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.01091, "step": 161, "tokens/total": 5264008, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 74083} +{"epoch": 0.6328125, "grad_norm": 0.7523948550224304, "learning_rate": 8.376452439121266e-05, "loss": 0.008357398211956024, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00839, "step": 162, "tokens/total": 5294720, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 74530} +{"epoch": 0.63671875, "grad_norm": 0.4489748775959015, "learning_rate": 8.354068477290124e-05, "loss": 0.017392598092556, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.01754, "step": 163, "tokens/total": 5325376, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 74999} +{"epoch": 0.640625, "grad_norm": 0.19750656187534332, "learning_rate": 8.331565746019807e-05, "loss": 0.0062455181032419205, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00627, "step": 164, "tokens/total": 5360240, "tokens/train_per_sec_per_gpu": 7.02, "tokens/trainable": 75450} +{"epoch": 0.64453125, "grad_norm": 0.43720558285713196, "learning_rate": 8.308945181740812e-05, "loss": 0.007017537485808134, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00704, "step": 165, "tokens/total": 5392376, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 75921} +{"epoch": 0.6484375, "grad_norm": 0.37282148003578186, "learning_rate": 8.286207725787153e-05, "loss": 0.015585815533995628, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.01571, "step": 166, "tokens/total": 5427456, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 76425} +{"epoch": 0.65234375, "grad_norm": 0.9922154545783997, "learning_rate": 8.263354324357182e-05, "loss": 0.04034440219402313, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.04117, "step": 167, "tokens/total": 5457688, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 76863} +{"epoch": 0.65625, "grad_norm": 0.24802115559577942, "learning_rate": 8.240385928474219e-05, "loss": 0.0026114522479474545, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00261, "step": 168, "tokens/total": 5488600, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 77298} +{"epoch": 0.66015625, "grad_norm": 1.317172646522522, "learning_rate": 8.217303493946967e-05, "loss": 0.008362943306565285, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.0084, "step": 169, "tokens/total": 5521856, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 77735} +{"epoch": 0.6640625, "grad_norm": 0.07680705189704895, "learning_rate": 8.194107981329746e-05, "loss": 0.0014382745139300823, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00144, "step": 170, "tokens/total": 5553272, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 78179} +{"epoch": 0.66796875, "grad_norm": 0.35469767451286316, "learning_rate": 8.170800355882518e-05, "loss": 0.0030405744910240173, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.57, "memory/max_allocated (GiB)": 65.57, "ppl": 1.00305, "step": 171, "tokens/total": 5587776, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 78660} +{"epoch": 0.671875, "grad_norm": 0.9873449802398682, "learning_rate": 8.147381587530713e-05, "loss": 0.004367139656096697, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00438, "step": 172, "tokens/total": 5620864, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 79110} +{"epoch": 0.67578125, "grad_norm": 1.3621207475662231, "learning_rate": 8.123852650824877e-05, "loss": 0.014000708237290382, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.0141, "step": 173, "tokens/total": 5653656, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 79576} +{"epoch": 0.6796875, "grad_norm": 0.01148117147386074, "learning_rate": 8.100214524900103e-05, "loss": 0.00025018281303346157, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00025, "step": 174, "tokens/total": 5682312, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 80001} +{"epoch": 0.68359375, "grad_norm": 1.0308685302734375, "learning_rate": 8.076468193435301e-05, "loss": 0.01147375162690878, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.01154, "step": 175, "tokens/total": 5713520, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 80452} +{"epoch": 0.6875, "grad_norm": 0.17354850471019745, "learning_rate": 8.052614644612253e-05, "loss": 0.00044286216143518686, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.00044, "step": 176, "tokens/total": 5745296, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 80886} +{"epoch": 0.69140625, "grad_norm": 2.6043660640716553, "learning_rate": 8.028654871074489e-05, "loss": 0.027438726276159286, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.02782, "step": 177, "tokens/total": 5774336, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 81325} +{"epoch": 0.6953125, "grad_norm": 3.082973003387451, "learning_rate": 8.004589869885986e-05, "loss": 0.008675106801092625, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00871, "step": 178, "tokens/total": 5805032, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 81732} +{"epoch": 0.69921875, "grad_norm": 0.12950804829597473, "learning_rate": 7.980420642489674e-05, "loss": 0.0011770488927140832, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00118, "step": 179, "tokens/total": 5832616, "tokens/train_per_sec_per_gpu": 11.16, "tokens/trainable": 82171} +{"epoch": 0.703125, "grad_norm": 0.6074830889701843, "learning_rate": 7.95614819466576e-05, "loss": 0.01626867800951004, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.0164, "step": 180, "tokens/total": 5865568, "tokens/train_per_sec_per_gpu": 10.49, "tokens/trainable": 82670} +{"epoch": 0.70703125, "grad_norm": 1.2389962673187256, "learning_rate": 7.931773536489872e-05, "loss": 0.0128059396520257, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.01289, "step": 181, "tokens/total": 5894256, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 83084} +{"epoch": 0.7109375, "grad_norm": 0.09234146773815155, "learning_rate": 7.907297682291035e-05, "loss": 0.0009862177539616823, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00099, "step": 182, "tokens/total": 5924304, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 83500} +{"epoch": 0.71484375, "grad_norm": 0.7711153626441956, "learning_rate": 7.882721650609442e-05, "loss": 0.005723479203879833, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00574, "step": 183, "tokens/total": 5955128, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 83896} +{"epoch": 0.71875, "grad_norm": 0.6413029432296753, "learning_rate": 7.85804646415409e-05, "loss": 0.008756429888308048, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.86, "memory/max_allocated (GiB)": 64.86, "ppl": 1.00879, "step": 184, "tokens/total": 5988056, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 84399} +{"epoch": 0.72265625, "grad_norm": 0.5943246483802795, "learning_rate": 7.833273149760207e-05, "loss": 0.015929536893963814, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.01606, "step": 185, "tokens/total": 6019544, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 84873} +{"epoch": 0.7265625, "grad_norm": 0.8045219779014587, "learning_rate": 7.808402738346527e-05, "loss": 0.02021005004644394, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.02042, "step": 186, "tokens/total": 6051616, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 85296} +{"epoch": 0.73046875, "grad_norm": 0.6206476092338562, "learning_rate": 7.783436264872382e-05, "loss": 0.006387989968061447, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.54, "memory/max_allocated (GiB)": 64.54, "ppl": 1.00641, "step": 187, "tokens/total": 6084304, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 85741} +{"epoch": 0.734375, "grad_norm": 0.22044190764427185, "learning_rate": 7.758374768294647e-05, "loss": 0.00326332775875926, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00327, "step": 188, "tokens/total": 6117896, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 86176} +{"epoch": 0.73828125, "grad_norm": 0.21264131367206573, "learning_rate": 7.733219291524489e-05, "loss": 0.0036738445051014423, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00368, "step": 189, "tokens/total": 6152296, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 86649} +{"epoch": 0.7421875, "grad_norm": 0.8155110478401184, "learning_rate": 7.707970881383977e-05, "loss": 0.0077774119563400745, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00781, "step": 190, "tokens/total": 6186056, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 87084} +{"epoch": 0.74609375, "grad_norm": 1.1746814250946045, "learning_rate": 7.682630588562518e-05, "loss": 0.021100454032421112, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.02132, "step": 191, "tokens/total": 6219160, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 87499} +{"epoch": 0.75, "grad_norm": 0.3209289610385895, "learning_rate": 7.657199467573129e-05, "loss": 0.0020590347703546286, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00206, "step": 192, "tokens/total": 6254840, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 87987} +{"epoch": 0.75390625, "grad_norm": 0.3175683915615082, "learning_rate": 7.631678576708561e-05, "loss": 0.00370378652587533, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00371, "step": 193, "tokens/total": 6289656, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 88470} +{"epoch": 0.7578125, "grad_norm": 0.6896700859069824, "learning_rate": 7.606068977997255e-05, "loss": 0.008992785587906837, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00903, "step": 194, "tokens/total": 6321776, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 88923} +{"epoch": 0.76171875, "grad_norm": 0.05941782146692276, "learning_rate": 7.580371737159148e-05, "loss": 0.0013049364788457751, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00131, "step": 195, "tokens/total": 6352824, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 89338} +{"epoch": 0.765625, "grad_norm": 0.18063241243362427, "learning_rate": 7.554587923561324e-05, "loss": 0.0030780513770878315, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00308, "step": 196, "tokens/total": 6388496, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 89812} +{"epoch": 0.76953125, "grad_norm": 0.6090373992919922, "learning_rate": 7.528718610173511e-05, "loss": 0.012562395073473454, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.01264, "step": 197, "tokens/total": 6420600, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 90252} +{"epoch": 0.7734375, "grad_norm": 1.0014491081237793, "learning_rate": 7.502764873523431e-05, "loss": 0.008216958492994308, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00825, "step": 198, "tokens/total": 6454616, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 90718} +{"epoch": 0.77734375, "grad_norm": 0.7299976944923401, "learning_rate": 7.476727793652011e-05, "loss": 0.007479894440621138, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00751, "step": 199, "tokens/total": 6485680, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 91194} +{"epoch": 0.78125, "grad_norm": 0.141410693526268, "learning_rate": 7.450608454068415e-05, "loss": 0.0016855563735589385, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00169, "step": 200, "tokens/total": 6518512, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 91649} +{"epoch": 0.78515625, "grad_norm": 0.5226508975028992, "learning_rate": 7.424407941704987e-05, "loss": 0.004424888640642166, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.00443, "step": 201, "tokens/total": 6548728, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 92078} +{"epoch": 0.7890625, "grad_norm": 0.16747581958770752, "learning_rate": 7.398127346871986e-05, "loss": 0.0021262692753225565, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00213, "step": 202, "tokens/total": 6582192, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 92536} +{"epoch": 0.79296875, "grad_norm": 0.32159551978111267, "learning_rate": 7.371767763212238e-05, "loss": 0.0024754926562309265, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.00248, "step": 203, "tokens/total": 6617392, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 93032} +{"epoch": 0.796875, "grad_norm": 0.15150900185108185, "learning_rate": 7.345330287655617e-05, "loss": 0.0012750524329021573, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00128, "step": 204, "tokens/total": 6648904, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 93504} +{"epoch": 0.80078125, "grad_norm": 0.3688977062702179, "learning_rate": 7.31881602037339e-05, "loss": 0.001986539689823985, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00199, "step": 205, "tokens/total": 6679832, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 93925} +{"epoch": 0.8046875, "grad_norm": 0.9076482653617859, "learning_rate": 7.29222606473245e-05, "loss": 0.00858451146632433, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00862, "step": 206, "tokens/total": 6710784, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 94365} +{"epoch": 0.80859375, "grad_norm": 0.18203569948673248, "learning_rate": 7.265561527249383e-05, "loss": 0.001240718411281705, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00124, "step": 207, "tokens/total": 6744848, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 94845} +{"epoch": 0.8125, "grad_norm": 1.4144891500473022, "learning_rate": 7.238823517544436e-05, "loss": 0.004370111972093582, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00438, "step": 208, "tokens/total": 6777600, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 95285} +{"epoch": 0.81640625, "grad_norm": 0.08326790481805801, "learning_rate": 7.212013148295333e-05, "loss": 0.000392957532312721, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00039, "step": 209, "tokens/total": 6811024, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 95758} +{"epoch": 0.8203125, "grad_norm": 0.9408327341079712, "learning_rate": 7.185131535190975e-05, "loss": 0.02043161168694496, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.15, "memory/max_allocated (GiB)": 66.15, "ppl": 1.02064, "step": 210, "tokens/total": 6845376, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 96192} +{"epoch": 0.82421875, "grad_norm": 1.650224208831787, "learning_rate": 7.158179796885005e-05, "loss": 0.01126706600189209, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01133, "step": 211, "tokens/total": 6880776, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 96641} +{"epoch": 0.828125, "grad_norm": 0.6798638105392456, "learning_rate": 7.131159054949273e-05, "loss": 0.018928037956357002, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.01911, "step": 212, "tokens/total": 6916160, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 97160} +{"epoch": 0.83203125, "grad_norm": 0.04109572246670723, "learning_rate": 7.104070433827139e-05, "loss": 0.00039244251092895865, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.54, "memory/max_allocated (GiB)": 63.54, "ppl": 1.00039, "step": 213, "tokens/total": 6945320, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 97611} +{"epoch": 0.8359375, "grad_norm": 0.3994831144809723, "learning_rate": 7.076915060786705e-05, "loss": 0.0031577215995639563, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00316, "step": 214, "tokens/total": 6977856, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 98111} +{"epoch": 0.83984375, "grad_norm": 0.43964797258377075, "learning_rate": 7.049694065873882e-05, "loss": 0.0020848819985985756, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.00209, "step": 215, "tokens/total": 7011096, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 98575} +{"epoch": 0.84375, "grad_norm": 0.05086313933134079, "learning_rate": 7.022408581865382e-05, "loss": 0.0006618685438297689, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.93, "memory/max_allocated (GiB)": 63.93, "ppl": 1.00066, "step": 216, "tokens/total": 7041408, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 98991} +{"epoch": 0.84765625, "grad_norm": 0.2872346043586731, "learning_rate": 6.99505974422157e-05, "loss": 0.003518722951412201, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.6, "memory/max_allocated (GiB)": 63.6, "ppl": 1.00352, "step": 217, "tokens/total": 7071560, "tokens/train_per_sec_per_gpu": 11.53, "tokens/trainable": 99454} +{"epoch": 0.8515625, "grad_norm": 0.004034165758639574, "learning_rate": 6.967648691039213e-05, "loss": 4.693739174399525e-05, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00005, "step": 218, "tokens/total": 7104696, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 99925} +{"epoch": 0.85546875, "grad_norm": 0.18472684919834137, "learning_rate": 6.940176563004123e-05, "loss": 0.0014208366628736258, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00142, "step": 219, "tokens/total": 7138504, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 100371} +{"epoch": 0.859375, "grad_norm": 0.5439212322235107, "learning_rate": 6.912644503343682e-05, "loss": 0.007101899944245815, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00713, "step": 220, "tokens/total": 7170992, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 100855} +{"epoch": 0.86328125, "grad_norm": 1.8649287223815918, "learning_rate": 6.885053657779273e-05, "loss": 0.002470525214448571, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00247, "step": 221, "tokens/total": 7201176, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 101253} +{"epoch": 0.8671875, "grad_norm": 0.602470338344574, "learning_rate": 6.857405174478604e-05, "loss": 0.003710217075422406, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00372, "step": 222, "tokens/total": 7233528, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 101706} +{"epoch": 0.87109375, "grad_norm": 0.8211750984191895, "learning_rate": 6.82970020400792e-05, "loss": 0.014173654839396477, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01427, "step": 223, "tokens/total": 7267672, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 102158} +{"epoch": 0.875, "grad_norm": 0.08592637628316879, "learning_rate": 6.801939899284132e-05, "loss": 0.0004529629077296704, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 62.86, "memory/max_allocated (GiB)": 62.86, "ppl": 1.00045, "step": 224, "tokens/total": 7295592, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 102580} +{"epoch": 0.87890625, "grad_norm": 0.4797177314758301, "learning_rate": 6.774125415526827e-05, "loss": 0.005428341217339039, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00544, "step": 225, "tokens/total": 7325816, "tokens/train_per_sec_per_gpu": 10.07, "tokens/trainable": 103063} +{"epoch": 0.8828125, "grad_norm": 2.7988901138305664, "learning_rate": 6.746257910210214e-05, "loss": 0.00956039596349001, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00961, "step": 226, "tokens/total": 7360888, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 103510} +{"epoch": 0.88671875, "grad_norm": 0.4000493884086609, "learning_rate": 6.718338543014937e-05, "loss": 0.0016584016848355532, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00166, "step": 227, "tokens/total": 7390416, "tokens/train_per_sec_per_gpu": 10.32, "tokens/trainable": 103980} +{"epoch": 0.890625, "grad_norm": 0.8817310929298401, "learning_rate": 6.69036847577983e-05, "loss": 0.010617760941386223, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.01067, "step": 228, "tokens/total": 7420480, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 104413} +{"epoch": 0.89453125, "grad_norm": 0.37602946162223816, "learning_rate": 6.662348872453553e-05, "loss": 0.011864264495670795, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.01193, "step": 229, "tokens/total": 7448992, "tokens/train_per_sec_per_gpu": 10.2, "tokens/trainable": 104845} +{"epoch": 0.8984375, "grad_norm": 0.6733375191688538, "learning_rate": 6.63428089904618e-05, "loss": 0.004317530896514654, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00433, "step": 230, "tokens/total": 7481744, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 105289} +{"epoch": 0.90234375, "grad_norm": 0.4976617991924286, "learning_rate": 6.60616572358065e-05, "loss": 0.011280687525868416, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01134, "step": 231, "tokens/total": 7516704, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 105722} +{"epoch": 0.90625, "grad_norm": 0.45896878838539124, "learning_rate": 6.578004516044172e-05, "loss": 0.009073503315448761, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.19, "memory/max_allocated (GiB)": 64.19, "ppl": 1.00911, "step": 232, "tokens/total": 7548592, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 106213} +{"epoch": 0.91015625, "grad_norm": 0.6351407170295715, "learning_rate": 6.549798448339548e-05, "loss": 0.007053213659673929, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00708, "step": 233, "tokens/total": 7579288, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 106643} +{"epoch": 0.9140625, "grad_norm": 0.8971742391586304, "learning_rate": 6.521548694236384e-05, "loss": 0.047368988394737244, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.04851, "step": 234, "tokens/total": 7611672, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 107136} +{"epoch": 0.91796875, "grad_norm": 0.5310046672821045, "learning_rate": 6.493256429322259e-05, "loss": 0.01087456289678812, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.01093, "step": 235, "tokens/total": 7646288, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 107588} +{"epoch": 0.921875, "grad_norm": 0.474041223526001, "learning_rate": 6.464922830953799e-05, "loss": 0.02061155065894127, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.02083, "step": 236, "tokens/total": 7680136, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 108087} +{"epoch": 0.92578125, "grad_norm": 1.2920523881912231, "learning_rate": 6.436549078207688e-05, "loss": 0.010183379985392094, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.61, "memory/max_allocated (GiB)": 64.61, "ppl": 1.01024, "step": 237, "tokens/total": 7713072, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 108533} +{"epoch": 0.9296875, "grad_norm": 0.11810195446014404, "learning_rate": 6.408136351831592e-05, "loss": 0.0035948343575000763, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.0036, "step": 238, "tokens/total": 7747856, "tokens/train_per_sec_per_gpu": 7.04, "tokens/trainable": 108965} +{"epoch": 0.93359375, "grad_norm": 0.33251866698265076, "learning_rate": 6.379685834195036e-05, "loss": 0.011865941807627678, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01194, "step": 239, "tokens/total": 7780136, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 109416} +{"epoch": 0.9375, "grad_norm": 0.3726855516433716, "learning_rate": 6.351198709240186e-05, "loss": 0.012707313522696495, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.78, "memory/max_allocated (GiB)": 65.78, "ppl": 1.01279, "step": 240, "tokens/total": 7816344, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 109864} +{"epoch": 0.94140625, "grad_norm": 0.29965031147003174, "learning_rate": 6.32267616243259e-05, "loss": 0.011894935742020607, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.01197, "step": 241, "tokens/total": 7850408, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 110345} +{"epoch": 0.9453125, "grad_norm": 0.23765899240970612, "learning_rate": 6.294119380711849e-05, "loss": 0.002795445965602994, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.0028, "step": 242, "tokens/total": 7881528, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 110794} +{"epoch": 0.94921875, "grad_norm": 0.098666250705719, "learning_rate": 6.265529552442209e-05, "loss": 0.0029243999160826206, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00293, "step": 243, "tokens/total": 7915712, "tokens/train_per_sec_per_gpu": 10.15, "tokens/trainable": 111250} +{"epoch": 0.953125, "grad_norm": 0.721277117729187, "learning_rate": 6.236907867363127e-05, "loss": 0.018348440527915955, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.61, "memory/max_allocated (GiB)": 64.61, "ppl": 1.01852, "step": 244, "tokens/total": 7947408, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 111693} +{"epoch": 0.95703125, "grad_norm": 0.5517546534538269, "learning_rate": 6.208255516539749e-05, "loss": 0.03319311514496803, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.26, "memory/max_allocated (GiB)": 65.26, "ppl": 1.03375, "step": 245, "tokens/total": 7979600, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 112158} +{"epoch": 0.9609375, "grad_norm": 1.0697840452194214, "learning_rate": 6.179573692313344e-05, "loss": 0.019521433860063553, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.01971, "step": 246, "tokens/total": 8010848, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 112601} +{"epoch": 0.96484375, "grad_norm": 0.10502277314662933, "learning_rate": 6.150863588251694e-05, "loss": 0.0021630506962537766, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.00217, "step": 247, "tokens/total": 8040632, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 113016} +{"epoch": 0.96875, "grad_norm": 0.5545222163200378, "learning_rate": 6.122126399099419e-05, "loss": 0.0024277709890156984, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00243, "step": 248, "tokens/total": 8072464, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 113476} +{"epoch": 0.97265625, "grad_norm": 0.1973181962966919, "learning_rate": 6.0933633207282615e-05, "loss": 0.0037132794968783855, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.96, "memory/max_allocated (GiB)": 64.96, "ppl": 1.00372, "step": 249, "tokens/total": 8106568, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 113941} +{"epoch": 0.9765625, "grad_norm": 0.2856753468513489, "learning_rate": 6.064575550087316e-05, "loss": 0.007798416074365377, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00783, "step": 250, "tokens/total": 8138496, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 114434} +{"epoch": 0.98046875, "grad_norm": 0.12166773527860641, "learning_rate": 6.0357642851532245e-05, "loss": 0.0024114721454679966, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00241, "step": 251, "tokens/total": 8168080, "tokens/train_per_sec_per_gpu": 10.41, "tokens/trainable": 114898} +{"epoch": 0.984375, "grad_norm": 0.027239495888352394, "learning_rate": 6.0069307248803294e-05, "loss": 0.0004454192239791155, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00045, "step": 252, "tokens/total": 8200888, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 115331} +{"epoch": 0.98828125, "grad_norm": 0.008783197961747646, "learning_rate": 5.9780760691507635e-05, "loss": 0.000257178267929703, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00026, "step": 253, "tokens/total": 8230704, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 115732} +{"epoch": 0.9921875, "grad_norm": 0.418956458568573, "learning_rate": 5.9492015187245334e-05, "loss": 0.005200427491217852, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00521, "step": 254, "tokens/total": 8266560, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 116221} +{"epoch": 0.99609375, "grad_norm": 0.3620906174182892, "learning_rate": 5.920308275189541e-05, "loss": 0.008293623104691505, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.19, "memory/max_allocated (GiB)": 65.19, "ppl": 1.00833, "step": 255, "tokens/total": 8300184, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 116684} +{"epoch": 1.0, "grad_norm": 1.1492022275924683, "learning_rate": 5.8913975409115874e-05, "loss": 0.015779471024870872, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.0159, "step": 256, "tokens/total": 8333624, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 117126} +{"epoch": 1.00390625, "grad_norm": 0.027434034273028374, "learning_rate": 5.8624705189843395e-05, "loss": 0.00039193141856230795, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00039, "step": 257, "tokens/total": 8364976, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 117592} +{"epoch": 1.0078125, "grad_norm": 0.3716159164905548, "learning_rate": 5.833528413179249e-05, "loss": 0.009423593990504742, "memory/device_reserved (GiB)": 66.62, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00947, "step": 258, "tokens/total": 8399568, "tokens/train_per_sec_per_gpu": 7.08, "tokens/trainable": 118030} +{"epoch": 1.01171875, "grad_norm": 0.6014235019683838, "learning_rate": 5.80457242789548e-05, "loss": 0.0066643948666751385, "memory/device_reserved (GiB)": 66.62, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00669, "step": 259, "tokens/total": 8430616, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 118470} +{"epoch": 1.015625, "grad_norm": 0.40719953179359436, "learning_rate": 5.77560376810977e-05, "loss": 0.01057388074696064, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.01063, "step": 260, "tokens/total": 8462696, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 118935} +{"epoch": 1.01953125, "grad_norm": 0.45710280537605286, "learning_rate": 5.7466236393263005e-05, "loss": 0.009139406494796276, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00918, "step": 261, "tokens/total": 8496152, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 119381} +{"epoch": 1.0234375, "grad_norm": 0.8830277323722839, "learning_rate": 5.717633247526522e-05, "loss": 0.027644537389278412, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.02803, "step": 262, "tokens/total": 8532912, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 119849} +{"epoch": 1.02734375, "grad_norm": 0.1540258526802063, "learning_rate": 5.688633799118971e-05, "loss": 0.0016823483165353537, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.00168, "step": 263, "tokens/total": 8565376, "tokens/train_per_sec_per_gpu": 10.01, "tokens/trainable": 120319} +{"epoch": 1.03125, "grad_norm": 0.31569904088974, "learning_rate": 5.659626500889066e-05, "loss": 0.008379202336072922, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00841, "step": 264, "tokens/total": 8597640, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 120804} +{"epoch": 1.03515625, "grad_norm": 0.03189004957675934, "learning_rate": 5.6306125599488905e-05, "loss": 0.0003974739811383188, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.0004, "step": 265, "tokens/total": 8627312, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 121233} +{"epoch": 1.0390625, "grad_norm": 0.29655930399894714, "learning_rate": 5.601593183686955e-05, "loss": 0.009636803530156612, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.87, "memory/max_allocated (GiB)": 64.87, "ppl": 1.00968, "step": 266, "tokens/total": 8659208, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 121671} +{"epoch": 1.04296875, "grad_norm": 0.3712855577468872, "learning_rate": 5.572569579717961e-05, "loss": 0.009871562011539936, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.00992, "step": 267, "tokens/total": 8693088, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 122141} +{"epoch": 1.046875, "grad_norm": 0.6174975037574768, "learning_rate": 5.543542955832538e-05, "loss": 0.008069916628301144, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.0081, "step": 268, "tokens/total": 8725576, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 122556} +{"epoch": 1.05078125, "grad_norm": 0.14115233719348907, "learning_rate": 5.514514519946986e-05, "loss": 0.0035871542058885098, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00359, "step": 269, "tokens/total": 8755192, "tokens/train_per_sec_per_gpu": 10.46, "tokens/trainable": 122988} +{"epoch": 1.0546875, "grad_norm": 0.06941018253564835, "learning_rate": 5.485485480053015e-05, "loss": 0.0017225848278030753, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00172, "step": 270, "tokens/total": 8787704, "tokens/train_per_sec_per_gpu": 10.72, "tokens/trainable": 123478} +{"epoch": 1.05859375, "grad_norm": 0.0558178573846817, "learning_rate": 5.4564570441674645e-05, "loss": 0.001127309980802238, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00113, "step": 271, "tokens/total": 8819768, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 123932} +{"epoch": 1.0625, "grad_norm": 0.26069945096969604, "learning_rate": 5.42743042028204e-05, "loss": 0.009280494414269924, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00932, "step": 272, "tokens/total": 8853352, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 124428} +{"epoch": 1.06640625, "grad_norm": 0.06119797006249428, "learning_rate": 5.3984068163130464e-05, "loss": 0.0012454005191102624, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00125, "step": 273, "tokens/total": 8886840, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 124866} +{"epoch": 1.0703125, "grad_norm": 0.07951291650533676, "learning_rate": 5.369387440051111e-05, "loss": 0.0013175024650990963, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00132, "step": 274, "tokens/total": 8920808, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 125324} +{"epoch": 1.07421875, "grad_norm": 0.18001216650009155, "learning_rate": 5.340373499110935e-05, "loss": 0.0018408342730253935, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00184, "step": 275, "tokens/total": 8955880, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 125813} +{"epoch": 1.078125, "grad_norm": 0.5473217368125916, "learning_rate": 5.3113662008810304e-05, "loss": 0.008348309434950352, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.00838, "step": 276, "tokens/total": 8989520, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 126252} +{"epoch": 1.08203125, "grad_norm": 0.06711679697036743, "learning_rate": 5.282366752473479e-05, "loss": 0.0015285417903214693, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00153, "step": 277, "tokens/total": 9024680, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 126696} +{"epoch": 1.0859375, "grad_norm": 0.2837786376476288, "learning_rate": 5.2533763606737005e-05, "loss": 0.009348972700536251, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00939, "step": 278, "tokens/total": 9056208, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 127173} +{"epoch": 1.08984375, "grad_norm": 0.8567968606948853, "learning_rate": 5.224396231890232e-05, "loss": 0.017095761373639107, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.01724, "step": 279, "tokens/total": 9086376, "tokens/train_per_sec_per_gpu": 10.11, "tokens/trainable": 127637} +{"epoch": 1.09375, "grad_norm": 0.12711340188980103, "learning_rate": 5.195427572104522e-05, "loss": 0.002131153829395771, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00213, "step": 280, "tokens/total": 9118944, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 128112} +{"epoch": 1.09765625, "grad_norm": 0.01125620398670435, "learning_rate": 5.166471586820751e-05, "loss": 0.0003357863752171397, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00034, "step": 281, "tokens/total": 9152920, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 128584} +{"epoch": 1.1015625, "grad_norm": 0.4228435754776001, "learning_rate": 5.1375294810156615e-05, "loss": 0.007568118162453175, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.0076, "step": 282, "tokens/total": 9185728, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 129051} +{"epoch": 1.10546875, "grad_norm": 0.01944291964173317, "learning_rate": 5.1086024590884144e-05, "loss": 0.00034641707316040993, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00035, "step": 283, "tokens/total": 9217984, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 129494} +{"epoch": 1.109375, "grad_norm": 0.03665037080645561, "learning_rate": 5.079691724810461e-05, "loss": 0.0005812338786199689, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00058, "step": 284, "tokens/total": 9253160, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 129983} +{"epoch": 1.11328125, "grad_norm": 0.24433745443820953, "learning_rate": 5.0507984812754684e-05, "loss": 0.003500321414321661, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00351, "step": 285, "tokens/total": 9286136, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 130405} +{"epoch": 1.1171875, "grad_norm": 0.39400696754455566, "learning_rate": 5.021923930849237e-05, "loss": 0.0024055286776274443, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00241, "step": 286, "tokens/total": 9318360, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 130829} +{"epoch": 1.12109375, "grad_norm": 0.41336822509765625, "learning_rate": 4.99306927511967e-05, "loss": 0.004316661041229963, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00433, "step": 287, "tokens/total": 9350288, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 131287} +{"epoch": 1.125, "grad_norm": 0.23754416406154633, "learning_rate": 4.964235714846775e-05, "loss": 0.002687842817977071, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00269, "step": 288, "tokens/total": 9382104, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 131712} +{"epoch": 1.12890625, "grad_norm": 0.02034550905227661, "learning_rate": 4.9354244499126866e-05, "loss": 0.00025303414440713823, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00025, "step": 289, "tokens/total": 9414392, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 132168} +{"epoch": 1.1328125, "grad_norm": 1.236604928970337, "learning_rate": 4.90663667927174e-05, "loss": 0.005312466993927956, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00533, "step": 290, "tokens/total": 9447112, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 132626} +{"epoch": 1.13671875, "grad_norm": 0.6239795684814453, "learning_rate": 4.877873600900581e-05, "loss": 0.005352644249796867, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00537, "step": 291, "tokens/total": 9483400, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 133119} +{"epoch": 1.140625, "grad_norm": 0.45933160185813904, "learning_rate": 4.849136411748306e-05, "loss": 0.006858489476144314, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.48, "memory/max_allocated (GiB)": 64.48, "ppl": 1.00688, "step": 292, "tokens/total": 9513896, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 133550} +{"epoch": 1.14453125, "grad_norm": 0.014953012578189373, "learning_rate": 4.8204263076866574e-05, "loss": 0.00015577170415781438, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00016, "step": 293, "tokens/total": 9550184, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 133992} +{"epoch": 1.1484375, "grad_norm": 0.2147570550441742, "learning_rate": 4.791744483460251e-05, "loss": 0.002932098228484392, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00294, "step": 294, "tokens/total": 9583920, "tokens/train_per_sec_per_gpu": 10.26, "tokens/trainable": 134467} +{"epoch": 1.15234375, "grad_norm": 0.09580613672733307, "learning_rate": 4.7630921326368736e-05, "loss": 0.0007366810459643602, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00074, "step": 295, "tokens/total": 9617288, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 134895} +{"epoch": 1.15625, "grad_norm": 0.8328028321266174, "learning_rate": 4.7344704475577916e-05, "loss": 0.005785539746284485, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 62.94, "memory/max_allocated (GiB)": 62.94, "ppl": 1.0058, "step": 296, "tokens/total": 9645600, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 135321} +{"epoch": 1.16015625, "grad_norm": 0.5334081053733826, "learning_rate": 4.705880619288153e-05, "loss": 0.006406291387975216, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00643, "step": 297, "tokens/total": 9679456, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 135765} +{"epoch": 1.1640625, "grad_norm": 0.056120023131370544, "learning_rate": 4.677323837567412e-05, "loss": 0.0004989947192370892, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.0005, "step": 298, "tokens/total": 9711696, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 136196} +{"epoch": 1.16796875, "grad_norm": 0.7540289163589478, "learning_rate": 4.6488012907598146e-05, "loss": 0.007644422352313995, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00767, "step": 299, "tokens/total": 9745792, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 136649} +{"epoch": 1.171875, "grad_norm": 0.014477689750492573, "learning_rate": 4.620314165804964e-05, "loss": 0.0002250641118735075, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00023, "step": 300, "tokens/total": 9780648, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 137128} +{"epoch": 1.17578125, "grad_norm": 0.30930185317993164, "learning_rate": 4.591863648168407e-05, "loss": 0.011461665853857994, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.01153, "step": 301, "tokens/total": 9813032, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 137585} +{"epoch": 1.1796875, "grad_norm": 1.0563206672668457, "learning_rate": 4.5634509217923135e-05, "loss": 0.007535598706454039, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00756, "step": 302, "tokens/total": 9845200, "tokens/train_per_sec_per_gpu": 10.2, "tokens/trainable": 138054} +{"epoch": 1.18359375, "grad_norm": 0.13565802574157715, "learning_rate": 4.535077169046201e-05, "loss": 0.0008476981311105192, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.29, "memory/max_allocated (GiB)": 65.29, "ppl": 1.00085, "step": 303, "tokens/total": 9878616, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 138559} +{"epoch": 1.1875, "grad_norm": 0.05525967478752136, "learning_rate": 4.506743570677743e-05, "loss": 0.0010449517285451293, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00105, "step": 304, "tokens/total": 9910776, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 139050} +{"epoch": 1.19140625, "grad_norm": 0.022223515436053276, "learning_rate": 4.478451305763618e-05, "loss": 0.0003199902712367475, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00032, "step": 305, "tokens/total": 9945480, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 139525} +{"epoch": 1.1953125, "grad_norm": 0.4657770097255707, "learning_rate": 4.450201551660454e-05, "loss": 0.0009623466175980866, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.00096, "step": 306, "tokens/total": 9975992, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 139969} +{"epoch": 1.19921875, "grad_norm": 0.09660152345895767, "learning_rate": 4.4219954839558276e-05, "loss": 0.0010499745840206742, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.00105, "step": 307, "tokens/total": 10006440, "tokens/train_per_sec_per_gpu": 10.07, "tokens/trainable": 140443} +{"epoch": 1.203125, "grad_norm": 0.016023218631744385, "learning_rate": 4.393834276419352e-05, "loss": 0.00018335907952859998, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00018, "step": 308, "tokens/total": 10042928, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 140926} +{"epoch": 1.20703125, "grad_norm": 0.006059701554477215, "learning_rate": 4.36571910095382e-05, "loss": 0.00012351800978649408, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00012, "step": 309, "tokens/total": 10075184, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 141344} +{"epoch": 1.2109375, "grad_norm": 0.28240153193473816, "learning_rate": 4.337651127546448e-05, "loss": 0.0034885143395513296, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00349, "step": 310, "tokens/total": 10106656, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 141753} +{"epoch": 1.21484375, "grad_norm": 0.35664311051368713, "learning_rate": 4.3096315242201736e-05, "loss": 0.0052544064819812775, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.82, "memory/max_allocated (GiB)": 63.82, "ppl": 1.00527, "step": 311, "tokens/total": 10138536, "tokens/train_per_sec_per_gpu": 10.23, "tokens/trainable": 142219} +{"epoch": 1.21875, "grad_norm": 0.039416126906871796, "learning_rate": 4.2816614569850635e-05, "loss": 0.000724918267223984, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.00073, "step": 312, "tokens/total": 10165768, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 142617} +{"epoch": 1.22265625, "grad_norm": 0.6550657153129578, "learning_rate": 4.2537420897897864e-05, "loss": 0.01181204803287983, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01188, "step": 313, "tokens/total": 10201624, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 143103} +{"epoch": 1.2265625, "grad_norm": 0.12665271759033203, "learning_rate": 4.225874584473174e-05, "loss": 0.0012904945760965347, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00129, "step": 314, "tokens/total": 10233304, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 143535} +{"epoch": 1.23046875, "grad_norm": 0.0504036508500576, "learning_rate": 4.19806010071587e-05, "loss": 0.0004442619683686644, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00044, "step": 315, "tokens/total": 10267904, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 144009} +{"epoch": 1.234375, "grad_norm": 2.502131223678589, "learning_rate": 4.170299795992081e-05, "loss": 0.013017533347010612, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.0131, "step": 316, "tokens/total": 10301280, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 144460} +{"epoch": 1.23828125, "grad_norm": 0.4830859899520874, "learning_rate": 4.142594825521398e-05, "loss": 0.005645636934787035, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00566, "step": 317, "tokens/total": 10337072, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 144936} +{"epoch": 1.2421875, "grad_norm": 0.022288916632533073, "learning_rate": 4.114946342220728e-05, "loss": 0.00017050004680640996, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.69, "memory/max_allocated (GiB)": 64.69, "ppl": 1.00017, "step": 318, "tokens/total": 10368688, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 145375} +{"epoch": 1.24609375, "grad_norm": 0.15959490835666656, "learning_rate": 4.087355496656321e-05, "loss": 0.002046356676146388, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00205, "step": 319, "tokens/total": 10400752, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 145849} +{"epoch": 1.25, "grad_norm": 0.035723067820072174, "learning_rate": 4.05982343699588e-05, "loss": 0.00029890419682487845, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.0003, "step": 320, "tokens/total": 10432112, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 146318} +{"epoch": 1.25390625, "grad_norm": 0.11799093335866928, "learning_rate": 4.0323513089607876e-05, "loss": 0.0010655872756615281, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00107, "step": 321, "tokens/total": 10465472, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 146821} +{"epoch": 1.2578125, "grad_norm": 0.14245568215847015, "learning_rate": 4.004940255778431e-05, "loss": 0.0008313163998536766, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00083, "step": 322, "tokens/total": 10496336, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 147285} +{"epoch": 1.26171875, "grad_norm": 0.02021217532455921, "learning_rate": 3.977591418134619e-05, "loss": 0.00014320030459202826, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00014, "step": 323, "tokens/total": 10525864, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 147729} +{"epoch": 1.265625, "grad_norm": 0.4678204655647278, "learning_rate": 3.95030593412612e-05, "loss": 0.008031392470002174, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00806, "step": 324, "tokens/total": 10559680, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 148169} +{"epoch": 1.26953125, "grad_norm": 0.009489459916949272, "learning_rate": 3.923084939213296e-05, "loss": 0.00018935903790406883, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00019, "step": 325, "tokens/total": 10595400, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 148630} +{"epoch": 1.2734375, "grad_norm": 0.10362693667411804, "learning_rate": 3.895929566172861e-05, "loss": 0.00044408836401998997, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00044, "step": 326, "tokens/total": 10626680, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 149063} +{"epoch": 1.27734375, "grad_norm": 0.32847344875335693, "learning_rate": 3.868840945050728e-05, "loss": 0.0017084815772250295, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00171, "step": 327, "tokens/total": 10661784, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 149549} +{"epoch": 1.28125, "grad_norm": 0.4571961760520935, "learning_rate": 3.841820203114995e-05, "loss": 0.014447949826717377, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.31, "memory/max_allocated (GiB)": 65.31, "ppl": 1.01455, "step": 328, "tokens/total": 10696184, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 150015} +{"epoch": 1.28515625, "grad_norm": 0.36645448207855225, "learning_rate": 3.814868464809027e-05, "loss": 0.000641914492007345, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00064, "step": 329, "tokens/total": 10729376, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 150514} +{"epoch": 1.2890625, "grad_norm": 0.0027508826460689306, "learning_rate": 3.787986851704667e-05, "loss": 4.162834011367522e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00004, "step": 330, "tokens/total": 10764824, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 151021} +{"epoch": 1.29296875, "grad_norm": 0.024318553507328033, "learning_rate": 3.7611764824555654e-05, "loss": 0.000251204299274832, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00025, "step": 331, "tokens/total": 10797072, "tokens/train_per_sec_per_gpu": 10.11, "tokens/trainable": 151486} +{"epoch": 1.296875, "grad_norm": 0.2504402697086334, "learning_rate": 3.734438472750619e-05, "loss": 0.002271349774673581, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00227, "step": 332, "tokens/total": 10829376, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 151956} +{"epoch": 1.30078125, "grad_norm": 0.010923619382083416, "learning_rate": 3.707773935267552e-05, "loss": 0.0002067407767754048, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00021, "step": 333, "tokens/total": 10863512, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 152442} +{"epoch": 1.3046875, "grad_norm": 0.13473805785179138, "learning_rate": 3.68118397962661e-05, "loss": 0.0010796214919537306, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00108, "step": 334, "tokens/total": 10894432, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 152890} +{"epoch": 1.30859375, "grad_norm": 0.2542962431907654, "learning_rate": 3.654669712344384e-05, "loss": 0.006872319616377354, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.34, "memory/max_allocated (GiB)": 63.34, "ppl": 1.0069, "step": 335, "tokens/total": 10924848, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 153368} +{"epoch": 1.3125, "grad_norm": 0.04805229604244232, "learning_rate": 3.628232236787763e-05, "loss": 0.00014039115922059864, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00014, "step": 336, "tokens/total": 10959024, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 153809} +{"epoch": 1.31640625, "grad_norm": 0.36261215806007385, "learning_rate": 3.6018726531280144e-05, "loss": 0.005175971891731024, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00519, "step": 337, "tokens/total": 10991088, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 154258} +{"epoch": 1.3203125, "grad_norm": 0.3277006149291992, "learning_rate": 3.575592058295017e-05, "loss": 0.0015439039561897516, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00155, "step": 338, "tokens/total": 11023904, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 154704} +{"epoch": 1.32421875, "grad_norm": 0.09325721859931946, "learning_rate": 3.549391545931585e-05, "loss": 0.0005667523946613073, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00057, "step": 339, "tokens/total": 11057680, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 155157} +{"epoch": 1.328125, "grad_norm": 0.37701424956321716, "learning_rate": 3.5232722063479914e-05, "loss": 0.004575102124363184, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00459, "step": 340, "tokens/total": 11089800, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 155582} +{"epoch": 1.33203125, "grad_norm": 0.008835740387439728, "learning_rate": 3.49723512647657e-05, "loss": 0.00013058319746050984, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00013, "step": 341, "tokens/total": 11117960, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 156047} +{"epoch": 1.3359375, "grad_norm": 0.005354566499590874, "learning_rate": 3.471281389826491e-05, "loss": 0.0001068726705852896, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00011, "step": 342, "tokens/total": 11151888, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 156500} +{"epoch": 1.33984375, "grad_norm": 0.33957064151763916, "learning_rate": 3.4454120764386764e-05, "loss": 0.0034570491407066584, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.22, "memory/max_allocated (GiB)": 65.22, "ppl": 1.00346, "step": 343, "tokens/total": 11184568, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 156931} +{"epoch": 1.34375, "grad_norm": 0.1743135154247284, "learning_rate": 3.4196282628408526e-05, "loss": 0.015785230323672295, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.01591, "step": 344, "tokens/total": 11216400, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 157385} +{"epoch": 1.34765625, "grad_norm": 0.001983094261959195, "learning_rate": 3.3939310220027456e-05, "loss": 4.589482705341652e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00005, "step": 345, "tokens/total": 11250768, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 157817} +{"epoch": 1.3515625, "grad_norm": 0.021473895758390427, "learning_rate": 3.3683214232914404e-05, "loss": 0.00027697632322087884, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00028, "step": 346, "tokens/total": 11281808, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 158273} +{"epoch": 1.35546875, "grad_norm": 0.06701363623142242, "learning_rate": 3.342800532426873e-05, "loss": 0.00035991723416373134, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00036, "step": 347, "tokens/total": 11313968, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 158728} +{"epoch": 1.359375, "grad_norm": 0.00447877449914813, "learning_rate": 3.317369411437484e-05, "loss": 5.736605089623481e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00006, "step": 348, "tokens/total": 11347288, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 159168} +{"epoch": 1.36328125, "grad_norm": 0.03362403064966202, "learning_rate": 3.292029118616024e-05, "loss": 0.00043672986794263124, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00044, "step": 349, "tokens/total": 11377128, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 159592} +{"epoch": 1.3671875, "grad_norm": 0.04746627062559128, "learning_rate": 3.266780708475511e-05, "loss": 0.0005799963837489486, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00058, "step": 350, "tokens/total": 11409488, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 160093} +{"epoch": 1.37109375, "grad_norm": 0.046278566122055054, "learning_rate": 3.241625231705354e-05, "loss": 0.00039735628524795175, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.0004, "step": 351, "tokens/total": 11442040, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 160534} +{"epoch": 1.375, "grad_norm": 0.10436023771762848, "learning_rate": 3.216563735127618e-05, "loss": 0.0007770842639729381, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00078, "step": 352, "tokens/total": 11478312, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 161009} +{"epoch": 1.37890625, "grad_norm": 0.04106978327035904, "learning_rate": 3.191597261653475e-05, "loss": 0.0005022163968533278, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.93, "memory/max_allocated (GiB)": 63.93, "ppl": 1.0005, "step": 353, "tokens/total": 11509008, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 161450} +{"epoch": 1.3828125, "grad_norm": 0.0585411973297596, "learning_rate": 3.166726850239794e-05, "loss": 0.0008533199434168637, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00085, "step": 354, "tokens/total": 11541264, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 161900} +{"epoch": 1.38671875, "grad_norm": 0.16496816277503967, "learning_rate": 3.141953535845912e-05, "loss": 0.0009116700384765863, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00091, "step": 355, "tokens/total": 11573072, "tokens/train_per_sec_per_gpu": 10.27, "tokens/trainable": 162358} +{"epoch": 1.390625, "grad_norm": 0.013530638068914413, "learning_rate": 3.11727834939056e-05, "loss": 0.00015971148968674242, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00016, "step": 356, "tokens/total": 11605224, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 162845} +{"epoch": 1.39453125, "grad_norm": 0.8127633333206177, "learning_rate": 3.092702317708967e-05, "loss": 0.015073633752763271, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.01519, "step": 357, "tokens/total": 11637512, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 163299} +{"epoch": 1.3984375, "grad_norm": 0.009652971290051937, "learning_rate": 3.0682264635101276e-05, "loss": 0.00015129921666812152, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00015, "step": 358, "tokens/total": 11671280, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 163774} +{"epoch": 1.40234375, "grad_norm": 0.02048882097005844, "learning_rate": 3.0438518053342407e-05, "loss": 0.00037954014260321856, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00038, "step": 359, "tokens/total": 11702008, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 164199} +{"epoch": 1.40625, "grad_norm": 0.7198262810707092, "learning_rate": 3.0195793575103266e-05, "loss": 0.006941205356270075, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00697, "step": 360, "tokens/total": 11732032, "tokens/train_per_sec_per_gpu": 10.18, "tokens/trainable": 164667} +{"epoch": 1.41015625, "grad_norm": 0.104706309735775, "learning_rate": 2.9954101301140146e-05, "loss": 0.0007968574063852429, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0008, "step": 361, "tokens/total": 11767472, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.31262847781181335, "learning_rate": 2.9713451289255123e-05, "loss": 0.002789740916341543, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00279, "step": 362, "tokens/total": 11798432, "tokens/train_per_sec_per_gpu": 10.64, "tokens/trainable": 165575} +{"epoch": 1.41796875, "grad_norm": 0.09182748943567276, "learning_rate": 2.9473853553877484e-05, "loss": 0.0008584990864619613, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00086, "step": 363, "tokens/total": 11833144, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 166060} +{"epoch": 1.421875, "grad_norm": 0.012046776711940765, "learning_rate": 2.9235318065647e-05, "loss": 0.00018889152852352709, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.00019, "step": 364, "tokens/total": 11865856, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 166478} +{"epoch": 1.42578125, "grad_norm": 0.07167322188615799, "learning_rate": 2.8997854750998964e-05, "loss": 0.0005192388780415058, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00052, "step": 365, "tokens/total": 11900400, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 166941} +{"epoch": 1.4296875, "grad_norm": 0.24809038639068604, "learning_rate": 2.8761473491751258e-05, "loss": 0.0037248688749969006, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.49, "memory/max_allocated (GiB)": 63.49, "ppl": 1.00373, "step": 366, "tokens/total": 11931368, "tokens/train_per_sec_per_gpu": 9.88, "tokens/trainable": 167381} +{"epoch": 1.43359375, "grad_norm": 0.05035989731550217, "learning_rate": 2.8526184124692883e-05, "loss": 0.0002732015564106405, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00027, "step": 367, "tokens/total": 11963320, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 167830} +{"epoch": 1.4375, "grad_norm": 0.008683479391038418, "learning_rate": 2.829199644117484e-05, "loss": 0.00015512768004555255, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00016, "step": 368, "tokens/total": 11997536, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 168266} +{"epoch": 1.44140625, "grad_norm": 0.1597283035516739, "learning_rate": 2.8058920186702553e-05, "loss": 0.0018920016009360552, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00189, "step": 369, "tokens/total": 12029528, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 168716} +{"epoch": 1.4453125, "grad_norm": 0.10719984024763107, "learning_rate": 2.782696506053033e-05, "loss": 0.0001840288023231551, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00018, "step": 370, "tokens/total": 12062504, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 169176} +{"epoch": 1.44921875, "grad_norm": 0.28118380904197693, "learning_rate": 2.7596140715257824e-05, "loss": 0.0007019875920377672, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.78, "memory/max_allocated (GiB)": 65.78, "ppl": 1.0007, "step": 371, "tokens/total": 12097376, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 169591} +{"epoch": 1.453125, "grad_norm": 0.058830276131629944, "learning_rate": 2.7366456756428184e-05, "loss": 0.0008206295897252858, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00082, "step": 372, "tokens/total": 12131920, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 170041} +{"epoch": 1.45703125, "grad_norm": 0.08238182961940765, "learning_rate": 2.7137922742128486e-05, "loss": 0.0005677985609509051, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00057, "step": 373, "tokens/total": 12164528, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 170516} +{"epoch": 1.4609375, "grad_norm": 0.029082054272294044, "learning_rate": 2.691054818259188e-05, "loss": 0.0002357538032811135, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.00024, "step": 374, "tokens/total": 12197376, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 170952} +{"epoch": 1.46484375, "grad_norm": 0.45691823959350586, "learning_rate": 2.6684342539801933e-05, "loss": 0.0014626127667725086, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00146, "step": 375, "tokens/total": 12233104, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 171424} +{"epoch": 1.46875, "grad_norm": 0.27490517497062683, "learning_rate": 2.645931522709877e-05, "loss": 0.0013041351921856403, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.69, "memory/max_allocated (GiB)": 65.69, "ppl": 1.0013, "step": 376, "tokens/total": 12269224, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 171884} +{"epoch": 1.47265625, "grad_norm": 0.0031961039640009403, "learning_rate": 2.6235475608787365e-05, "loss": 5.423064067144878e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00005, "step": 377, "tokens/total": 12302952, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 172404} +{"epoch": 1.4765625, "grad_norm": 0.04778604209423065, "learning_rate": 2.6012832999747916e-05, "loss": 0.0004676888056565076, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00047, "step": 378, "tokens/total": 12338424, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 172871} +{"epoch": 1.48046875, "grad_norm": 0.37414270639419556, "learning_rate": 2.579139666504821e-05, "loss": 0.0035277255810797215, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00353, "step": 379, "tokens/total": 12369144, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 173329} +{"epoch": 1.484375, "grad_norm": 0.14965635538101196, "learning_rate": 2.557117581955798e-05, "loss": 0.0005219504237174988, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00052, "step": 380, "tokens/total": 12403352, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 173827} +{"epoch": 1.48828125, "grad_norm": 0.02740643359720707, "learning_rate": 2.5352179627565532e-05, "loss": 0.00023629216593690217, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00024, "step": 381, "tokens/total": 12441056, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 174287} +{"epoch": 1.4921875, "grad_norm": 0.10640553385019302, "learning_rate": 2.5134417202396277e-05, "loss": 0.0005714560393244028, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.00057, "step": 382, "tokens/total": 12474080, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 174731} +{"epoch": 1.49609375, "grad_norm": 0.050887152552604675, "learning_rate": 2.491789760603361e-05, "loss": 0.00021820103575009853, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00022, "step": 383, "tokens/total": 12508992, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 175183} +{"epoch": 1.5, "grad_norm": 0.008097662590444088, "learning_rate": 2.4702629848741764e-05, "loss": 0.0001258087286259979, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00013, "step": 384, "tokens/total": 12540896, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 175655} +{"epoch": 1.50390625, "grad_norm": 0.005564351566135883, "learning_rate": 2.4488622888690785e-05, "loss": 0.00011759593326132745, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00012, "step": 385, "tokens/total": 12568416, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 176085} +{"epoch": 1.5078125, "grad_norm": 0.010811757296323776, "learning_rate": 2.427588563158384e-05, "loss": 5.376443368731998e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00005, "step": 386, "tokens/total": 12599792, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 176549} +{"epoch": 1.51171875, "grad_norm": 0.06767824292182922, "learning_rate": 2.406442693028651e-05, "loss": 0.0005420631496235728, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00054, "step": 387, "tokens/total": 12631224, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 176995} +{"epoch": 1.515625, "grad_norm": 0.13407263159751892, "learning_rate": 2.3854255584458547e-05, "loss": 0.0006834836094640195, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00068, "step": 388, "tokens/total": 12667240, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 177468} +{"epoch": 1.51953125, "grad_norm": 0.012716291472315788, "learning_rate": 2.3645380340187508e-05, "loss": 7.596719660796225e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.54, "memory/max_allocated (GiB)": 65.54, "ppl": 1.00008, "step": 389, "tokens/total": 12700488, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 177961} +{"epoch": 1.5234375, "grad_norm": 0.012773267924785614, "learning_rate": 2.3437809889624914e-05, "loss": 7.853990973671898e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.00008, "step": 390, "tokens/total": 12734528, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 178417} +{"epoch": 1.52734375, "grad_norm": 0.0748896598815918, "learning_rate": 2.3231552870624487e-05, "loss": 0.0002991098153870553, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.0003, "step": 391, "tokens/total": 12767152, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 178862} +{"epoch": 1.53125, "grad_norm": 0.01770074851810932, "learning_rate": 2.3026617866382657e-05, "loss": 0.00021177723829168826, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.00021, "step": 392, "tokens/total": 12798168, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 179320} +{"epoch": 1.53515625, "grad_norm": 0.013511384837329388, "learning_rate": 2.2823013405081507e-05, "loss": 0.0001928990677697584, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00019, "step": 393, "tokens/total": 12828896, "tokens/train_per_sec_per_gpu": 10.06, "tokens/trainable": 179758} +{"epoch": 1.5390625, "grad_norm": 0.007867004722356796, "learning_rate": 2.2620747959533722e-05, "loss": 7.496929174521938e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00007, "step": 394, "tokens/total": 12860024, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 180211} +{"epoch": 1.54296875, "grad_norm": 1.1248297691345215, "learning_rate": 2.2419829946830123e-05, "loss": 0.011123976670205593, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01119, "step": 395, "tokens/total": 12891472, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 180637} +{"epoch": 1.546875, "grad_norm": 0.004170657601207495, "learning_rate": 2.2220267727989325e-05, "loss": 3.842132355202921e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00004, "step": 396, "tokens/total": 12925648, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 181142} +{"epoch": 1.55078125, "grad_norm": 0.2399294376373291, "learning_rate": 2.202206960760984e-05, "loss": 0.0015540622407570481, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.77, "memory/max_allocated (GiB)": 64.77, "ppl": 1.00156, "step": 397, "tokens/total": 12959200, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 181597} +{"epoch": 1.5546875, "grad_norm": 0.005224838387221098, "learning_rate": 2.182524383352446e-05, "loss": 7.990784069988877e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00008, "step": 398, "tokens/total": 12989624, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 182030} +{"epoch": 1.55859375, "grad_norm": 0.022487085312604904, "learning_rate": 2.1629798596457056e-05, "loss": 2.3493998014600947e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 62.97, "memory/max_allocated (GiB)": 62.97, "ppl": 1.00002, "step": 399, "tokens/total": 13019040, "tokens/train_per_sec_per_gpu": 10.48, "tokens/trainable": 182496} +{"epoch": 1.5625, "grad_norm": 0.05306445062160492, "learning_rate": 2.1435742029681725e-05, "loss": 0.00014889906742610037, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00015, "step": 400, "tokens/total": 13051808, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 182963} +{"epoch": 1.56640625, "grad_norm": 0.005385732278227806, "learning_rate": 2.124308220868431e-05, "loss": 6.0735521401511505e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00006, "step": 401, "tokens/total": 13085520, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 183416} +{"epoch": 1.5703125, "grad_norm": 0.0016650495817884803, "learning_rate": 2.105182715082638e-05, "loss": 3.281393583165482e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00003, "step": 402, "tokens/total": 13116224, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 183857} +{"epoch": 1.57421875, "grad_norm": 0.2708057165145874, "learning_rate": 2.0861984815011552e-05, "loss": 0.005137327127158642, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00515, "step": 403, "tokens/total": 13149984, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 184296} +{"epoch": 1.578125, "grad_norm": 0.47593140602111816, "learning_rate": 2.0673563101354323e-05, "loss": 0.0039229183457791805, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00393, "step": 404, "tokens/total": 13185488, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 184719} +{"epoch": 1.58203125, "grad_norm": 0.12336844950914383, "learning_rate": 2.0486569850851317e-05, "loss": 0.0003344530414324254, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00033, "step": 405, "tokens/total": 13216320, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 185172} +{"epoch": 1.5859375, "grad_norm": 0.039270397275686264, "learning_rate": 2.0301012845054956e-05, "loss": 0.00024725327966734767, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00025, "step": 406, "tokens/total": 13248136, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 185646} +{"epoch": 1.58984375, "grad_norm": 0.013352387584745884, "learning_rate": 2.011689980574966e-05, "loss": 9.420893184142187e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00009, "step": 407, "tokens/total": 13280640, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 186129} +{"epoch": 1.59375, "grad_norm": 0.17503955960273743, "learning_rate": 1.993423839463052e-05, "loss": 0.0006124452920630574, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00061, "step": 408, "tokens/total": 13314160, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 186610} +{"epoch": 1.59765625, "grad_norm": 0.4147596061229706, "learning_rate": 1.975303621298445e-05, "loss": 0.006095043383538723, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.51, "memory/max_allocated (GiB)": 63.51, "ppl": 1.00611, "step": 409, "tokens/total": 13345024, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 187066} +{"epoch": 1.6015625, "grad_norm": 0.005658351816236973, "learning_rate": 1.957330080137385e-05, "loss": 7.65794757171534e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00008, "step": 410, "tokens/total": 13377248, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 187567} +{"epoch": 1.60546875, "grad_norm": 0.07082543522119522, "learning_rate": 1.9395039639322864e-05, "loss": 0.000679413671605289, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00068, "step": 411, "tokens/total": 13410992, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 188056} +{"epoch": 1.609375, "grad_norm": 0.0033412850461900234, "learning_rate": 1.9218260145006073e-05, "loss": 3.505852509988472e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00004, "step": 412, "tokens/total": 13442728, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 188502} +{"epoch": 1.61328125, "grad_norm": 0.01695338822901249, "learning_rate": 1.904296967493982e-05, "loss": 0.0001344632328255102, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00013, "step": 413, "tokens/total": 13474824, "tokens/train_per_sec_per_gpu": 10.2, "tokens/trainable": 188995} +{"epoch": 1.6171875, "grad_norm": 0.784920334815979, "learning_rate": 1.8869175523676064e-05, "loss": 0.00818417314440012, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00822, "step": 414, "tokens/total": 13506528, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 189483} +{"epoch": 1.62109375, "grad_norm": 0.174355611205101, "learning_rate": 1.869688492349885e-05, "loss": 0.0014904497656971216, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00149, "step": 415, "tokens/total": 13537792, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 189935} +{"epoch": 1.625, "grad_norm": 0.03823116421699524, "learning_rate": 1.85261050441233e-05, "loss": 0.00020177855913061649, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.0002, "step": 416, "tokens/total": 13573136, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 190399} +{"epoch": 1.62890625, "grad_norm": 0.07657240331172943, "learning_rate": 1.8356842992397304e-05, "loss": 0.0007513907621614635, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00075, "step": 417, "tokens/total": 13607240, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 190862} +{"epoch": 1.6328125, "grad_norm": 0.004371723160147667, "learning_rate": 1.8189105812005714e-05, "loss": 3.789143738686107e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00004, "step": 418, "tokens/total": 13638920, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 191294} +{"epoch": 1.63671875, "grad_norm": 0.1552884727716446, "learning_rate": 1.802290048317732e-05, "loss": 0.0016698000254109502, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00167, "step": 419, "tokens/total": 13672320, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 191764} +{"epoch": 1.640625, "grad_norm": 0.054848358035087585, "learning_rate": 1.785823392239424e-05, "loss": 0.0003193699521943927, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.00032, "step": 420, "tokens/total": 13703312, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 192196} +{"epoch": 1.64453125, "grad_norm": 0.07637980580329895, "learning_rate": 1.7695112982104225e-05, "loss": 0.0005662624607793987, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00057, "step": 421, "tokens/total": 13734440, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 192612} +{"epoch": 1.6484375, "grad_norm": 0.0026949576567858458, "learning_rate": 1.7533544450435433e-05, "loss": 4.466508107725531e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.62, "memory/max_allocated (GiB)": 64.62, "ppl": 1.00004, "step": 422, "tokens/total": 13766944, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 193059} +{"epoch": 1.65234375, "grad_norm": 0.08674374967813492, "learning_rate": 1.7373535050913946e-05, "loss": 0.0007421313202939928, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00074, "step": 423, "tokens/total": 13797120, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 193466} +{"epoch": 1.65625, "grad_norm": 0.0031889318488538265, "learning_rate": 1.721509144218405e-05, "loss": 4.320424341131002e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00004, "step": 424, "tokens/total": 13828040, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 193910} +{"epoch": 1.66015625, "grad_norm": 0.6322300434112549, "learning_rate": 1.705822021773101e-05, "loss": 0.0037543789949268103, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.06, "memory/max_allocated (GiB)": 63.06, "ppl": 1.00376, "step": 425, "tokens/total": 13856328, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 194314} +{"epoch": 1.6640625, "grad_norm": 0.023231057450175285, "learning_rate": 1.69029279056068e-05, "loss": 0.0001322885655099526, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00013, "step": 426, "tokens/total": 13891312, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 194782} +{"epoch": 1.66796875, "grad_norm": 0.21323451399803162, "learning_rate": 1.6749220968158415e-05, "loss": 0.001494969823397696, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.0015, "step": 427, "tokens/total": 13925272, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 195254} +{"epoch": 1.671875, "grad_norm": 0.0021179693285375834, "learning_rate": 1.659710580175893e-05, "loss": 3.7042656913399696e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.53, "memory/max_allocated (GiB)": 63.53, "ppl": 1.00004, "step": 428, "tokens/total": 13955752, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 195702} +{"epoch": 1.67578125, "grad_norm": 0.1636502891778946, "learning_rate": 1.644658873654133e-05, "loss": 0.0012008484918624163, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.0012, "step": 429, "tokens/total": 13988000, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 196164} +{"epoch": 1.6796875, "grad_norm": 0.17666833102703094, "learning_rate": 1.629767603613508e-05, "loss": 0.0009825623128563166, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00098, "step": 430, "tokens/total": 14018712, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 196604} +{"epoch": 1.68359375, "grad_norm": 0.006396119482815266, "learning_rate": 1.615037389740547e-05, "loss": 7.37648515496403e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00007, "step": 431, "tokens/total": 14051992, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 197070} +{"epoch": 1.6875, "grad_norm": 0.002490109298378229, "learning_rate": 1.600468845019576e-05, "loss": 3.248464781790972e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.36, "memory/max_allocated (GiB)": 63.36, "ppl": 1.00003, "step": 432, "tokens/total": 14082624, "tokens/train_per_sec_per_gpu": 10.55, "tokens/trainable": 197552} +{"epoch": 1.69140625, "grad_norm": 0.548003613948822, "learning_rate": 1.5860625757072092e-05, "loss": 0.00313183874823153, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00314, "step": 433, "tokens/total": 14113304, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 197990} +{"epoch": 1.6953125, "grad_norm": 0.03978642448782921, "learning_rate": 1.571819181307116e-05, "loss": 0.00047346350038424134, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00047, "step": 434, "tokens/total": 14149592, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 198513} +{"epoch": 1.69921875, "grad_norm": 0.04031076282262802, "learning_rate": 1.557739254545075e-05, "loss": 0.0004019344924017787, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.0004, "step": 435, "tokens/total": 14183584, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 198979} +{"epoch": 1.703125, "grad_norm": 0.0218765027821064, "learning_rate": 1.543823381344311e-05, "loss": 0.00023763607896398753, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00024, "step": 436, "tokens/total": 14213688, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 199443} +{"epoch": 1.70703125, "grad_norm": 0.009754578582942486, "learning_rate": 1.5300721408011114e-05, "loss": 5.686923395842314e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.63, "memory/max_allocated (GiB)": 65.63, "ppl": 1.00006, "step": 437, "tokens/total": 14246480, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 199899} +{"epoch": 1.7109375, "grad_norm": 0.29929283261299133, "learning_rate": 1.5164861051607254e-05, "loss": 0.003880981123074889, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00389, "step": 438, "tokens/total": 14278320, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 200368} +{"epoch": 1.71484375, "grad_norm": 0.006473569665104151, "learning_rate": 1.5030658397935521e-05, "loss": 6.364710134221241e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00006, "step": 439, "tokens/total": 14313016, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 200819} +{"epoch": 1.71875, "grad_norm": 0.17386597394943237, "learning_rate": 1.4898119031716104e-05, "loss": 0.0009527600486762822, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00095, "step": 440, "tokens/total": 14343352, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 201275} +{"epoch": 1.72265625, "grad_norm": 0.018738625571131706, "learning_rate": 1.476724846845306e-05, "loss": 0.00018166808877140284, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00018, "step": 441, "tokens/total": 14377736, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 201699} +{"epoch": 1.7265625, "grad_norm": 0.04151351377367973, "learning_rate": 1.463805215420471e-05, "loss": 0.00024037616094574332, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00024, "step": 442, "tokens/total": 14415056, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 202200} +{"epoch": 1.73046875, "grad_norm": 0.6086709499359131, "learning_rate": 1.451053546535705e-05, "loss": 0.014220709912478924, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.01432, "step": 443, "tokens/total": 14449544, "tokens/train_per_sec_per_gpu": 7.08, "tokens/trainable": 202640} +{"epoch": 1.734375, "grad_norm": 0.005202199332416058, "learning_rate": 1.438470370840001e-05, "loss": 3.3660046028671786e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00003, "step": 444, "tokens/total": 14483472, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 203122} +{"epoch": 1.73828125, "grad_norm": 0.004539648070931435, "learning_rate": 1.4260562119706606e-05, "loss": 3.452198870945722e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.19, "memory/max_allocated (GiB)": 65.19, "ppl": 1.00003, "step": 445, "tokens/total": 14517648, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 203587} +{"epoch": 1.7421875, "grad_norm": 0.0063036601059138775, "learning_rate": 1.413811586531508e-05, "loss": 7.885813829489052e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00008, "step": 446, "tokens/total": 14553056, "tokens/train_per_sec_per_gpu": 6.86, "tokens/trainable": 204024} +{"epoch": 1.74609375, "grad_norm": 0.07461582124233246, "learning_rate": 1.4017370040713884e-05, "loss": 0.0005455457721836865, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.00055, "step": 447, "tokens/total": 14584280, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 204465} +{"epoch": 1.75, "grad_norm": 0.6728899478912354, "learning_rate": 1.3898329670629645e-05, "loss": 0.0012786773731932044, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00128, "step": 448, "tokens/total": 14618320, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 204923} +{"epoch": 1.75390625, "grad_norm": 0.2399381399154663, "learning_rate": 1.3780999708818058e-05, "loss": 0.0006735935457982123, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00067, "step": 449, "tokens/total": 14651264, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 205388} +{"epoch": 1.7578125, "grad_norm": 0.004919261205941439, "learning_rate": 1.3665385037857758e-05, "loss": 7.133408507797867e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00007, "step": 450, "tokens/total": 14682896, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 205873} +{"epoch": 1.76171875, "grad_norm": 0.8692010641098022, "learning_rate": 1.3551490468947126e-05, "loss": 0.0031273551285266876, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00313, "step": 451, "tokens/total": 14715272, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 206346} +{"epoch": 1.765625, "grad_norm": 0.30429574847221375, "learning_rate": 1.3439320741704075e-05, "loss": 0.00429863715544343, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00431, "step": 452, "tokens/total": 14744840, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 206776} +{"epoch": 1.76953125, "grad_norm": 0.011777542531490326, "learning_rate": 1.3328880523968808e-05, "loss": 6.183774530654773e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00006, "step": 453, "tokens/total": 14778008, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 207233} +{"epoch": 1.7734375, "grad_norm": 0.0033786462154239416, "learning_rate": 1.3220174411609587e-05, "loss": 4.583235931931995e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00005, "step": 454, "tokens/total": 14808472, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 207695} +{"epoch": 1.77734375, "grad_norm": 0.00047887954860925674, "learning_rate": 1.3113206928331471e-05, "loss": 1.1427468052715994e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00001, "step": 455, "tokens/total": 14841048, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 208144} +{"epoch": 1.78125, "grad_norm": 0.002935383701696992, "learning_rate": 1.300798252548806e-05, "loss": 4.1728657379280776e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00004, "step": 456, "tokens/total": 14876904, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 208602} +{"epoch": 1.78515625, "grad_norm": 0.22923657298088074, "learning_rate": 1.2904505581896265e-05, "loss": 0.0022206397261470556, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.58, "memory/max_allocated (GiB)": 64.58, "ppl": 1.00222, "step": 457, "tokens/total": 14909512, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 209040} +{"epoch": 1.7890625, "grad_norm": 0.989880383014679, "learning_rate": 1.2802780403654082e-05, "loss": 0.009888088330626488, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00994, "step": 458, "tokens/total": 14942048, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 209501} +{"epoch": 1.79296875, "grad_norm": 0.00997136440128088, "learning_rate": 1.2702811223961408e-05, "loss": 0.00012087162758689374, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.00012, "step": 459, "tokens/total": 14977032, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 209993} +{"epoch": 1.796875, "grad_norm": 0.6955318450927734, "learning_rate": 1.2604602202943861e-05, "loss": 0.009863144718110561, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00991, "step": 460, "tokens/total": 15011904, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 210455} +{"epoch": 1.80078125, "grad_norm": 0.049224428832530975, "learning_rate": 1.2508157427479686e-05, "loss": 0.0005093859508633614, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00051, "step": 461, "tokens/total": 15045504, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 210960} +{"epoch": 1.8046875, "grad_norm": 0.3815789520740509, "learning_rate": 1.2413480911029655e-05, "loss": 0.002933244686573744, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.15, "memory/max_allocated (GiB)": 66.15, "ppl": 1.00294, "step": 462, "tokens/total": 15078536, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 211405} +{"epoch": 1.80859375, "grad_norm": 0.007990461774170399, "learning_rate": 1.2320576593470082e-05, "loss": 8.999522833619267e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00009, "step": 463, "tokens/total": 15111640, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 211881} +{"epoch": 1.8125, "grad_norm": 0.3913213014602661, "learning_rate": 1.2229448340928828e-05, "loss": 0.007567425258457661, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.0076, "step": 464, "tokens/total": 15142608, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 212338} +{"epoch": 1.81640625, "grad_norm": 0.014464995823800564, "learning_rate": 1.2140099945624458e-05, "loss": 0.00011262228508712724, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.00011, "step": 465, "tokens/total": 15175048, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 212803} +{"epoch": 1.8203125, "grad_norm": 0.005938629154115915, "learning_rate": 1.205253512570841e-05, "loss": 7.986863784026355e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00008, "step": 466, "tokens/total": 15205776, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 213215} +{"epoch": 1.82421875, "grad_norm": 0.004686491563916206, "learning_rate": 1.1966757525110255e-05, "loss": 6.723435944877565e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.00007, "step": 467, "tokens/total": 15239240, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 213675} +{"epoch": 1.828125, "grad_norm": 0.054294753819704056, "learning_rate": 1.1882770713386095e-05, "loss": 0.000405237078666687, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00041, "step": 468, "tokens/total": 15273696, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 214169} +{"epoch": 1.83203125, "grad_norm": 0.0019386201165616512, "learning_rate": 1.180057818556998e-05, "loss": 3.4845677873818204e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00003, "step": 469, "tokens/total": 15307632, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 214624} +{"epoch": 1.8359375, "grad_norm": 0.5268276333808899, "learning_rate": 1.1720183362028494e-05, "loss": 0.008049475029110909, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00808, "step": 470, "tokens/total": 15339960, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 215087} +{"epoch": 1.83984375, "grad_norm": 0.2253703474998474, "learning_rate": 1.1641589588318387e-05, "loss": 0.001745249843224883, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00175, "step": 471, "tokens/total": 15373640, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 215553} +{"epoch": 1.84375, "grad_norm": 0.06184569001197815, "learning_rate": 1.1564800135047418e-05, "loss": 0.0003525453503243625, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00035, "step": 472, "tokens/total": 15406304, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 216018} +{"epoch": 1.84765625, "grad_norm": 0.0030149586964398623, "learning_rate": 1.148981819773816e-05, "loss": 5.58269202883821e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00006, "step": 473, "tokens/total": 15440128, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 216477} +{"epoch": 1.8515625, "grad_norm": 0.25112468004226685, "learning_rate": 1.1416646896695086e-05, "loss": 0.003493361175060272, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.0035, "step": 474, "tokens/total": 15471600, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 216949} +{"epoch": 1.85546875, "grad_norm": 0.12697680294513702, "learning_rate": 1.1345289276874717e-05, "loss": 0.00093157246010378, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00093, "step": 475, "tokens/total": 15507016, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 217408} +{"epoch": 1.859375, "grad_norm": 0.3983783423900604, "learning_rate": 1.1275748307758873e-05, "loss": 0.006720948498696089, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00674, "step": 476, "tokens/total": 15542600, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 217854} +{"epoch": 1.86328125, "grad_norm": 0.10787630081176758, "learning_rate": 1.1208026883231147e-05, "loss": 0.0006201534997671843, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00062, "step": 477, "tokens/total": 15577320, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 218337} +{"epoch": 1.8671875, "grad_norm": 0.014260904863476753, "learning_rate": 1.1142127821456433e-05, "loss": 0.00014325589290820062, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00014, "step": 478, "tokens/total": 15608840, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 218788} +{"epoch": 1.87109375, "grad_norm": 0.004564950708299875, "learning_rate": 1.1078053864763674e-05, "loss": 6.76714553264901e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00007, "step": 479, "tokens/total": 15640160, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 219219} +{"epoch": 1.875, "grad_norm": 0.07089844346046448, "learning_rate": 1.1015807679531756e-05, "loss": 0.0006772764609195292, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00068, "step": 480, "tokens/total": 15673920, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 219676} +{"epoch": 1.87890625, "grad_norm": 0.06665774434804916, "learning_rate": 1.0955391856078528e-05, "loss": 0.00034615685581229627, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00035, "step": 481, "tokens/total": 15705512, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 220125} +{"epoch": 1.8828125, "grad_norm": 0.03149803355336189, "learning_rate": 1.0896808908553007e-05, "loss": 0.00032934837508946657, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00033, "step": 482, "tokens/total": 15742096, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 220615} +{"epoch": 1.88671875, "grad_norm": 0.03199896961450577, "learning_rate": 1.0840061274830763e-05, "loss": 0.0002484047436155379, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00025, "step": 483, "tokens/total": 15773064, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 221081} +{"epoch": 1.890625, "grad_norm": 0.04643042013049126, "learning_rate": 1.0785151316412473e-05, "loss": 0.00017820039647631347, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00018, "step": 484, "tokens/total": 15805368, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 221546} +{"epoch": 1.89453125, "grad_norm": 0.014955771155655384, "learning_rate": 1.0732081318325639e-05, "loss": 0.00013466030941344798, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.57, "memory/max_allocated (GiB)": 65.57, "ppl": 1.00013, "step": 485, "tokens/total": 15839480, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 222014} +{"epoch": 1.8984375, "grad_norm": 0.02068273536860943, "learning_rate": 1.0680853489029501e-05, "loss": 0.00027406771550886333, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00027, "step": 486, "tokens/total": 15870672, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 222446} +{"epoch": 1.90234375, "grad_norm": 0.32149460911750793, "learning_rate": 1.0631469960323152e-05, "loss": 0.001389186130836606, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.00139, "step": 487, "tokens/total": 15902784, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 222899} +{"epoch": 1.90625, "grad_norm": 0.016509104520082474, "learning_rate": 1.0583932787256783e-05, "loss": 0.000210450918530114, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00021, "step": 488, "tokens/total": 15935456, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 223333} +{"epoch": 1.91015625, "grad_norm": 0.07668687403202057, "learning_rate": 1.0538243948046206e-05, "loss": 0.0004974472685717046, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.0005, "step": 489, "tokens/total": 15967960, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 223780} +{"epoch": 1.9140625, "grad_norm": 0.21462741494178772, "learning_rate": 1.0494405343990523e-05, "loss": 0.004225879907608032, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00423, "step": 490, "tokens/total": 16003904, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 224253} +{"epoch": 1.91796875, "grad_norm": 0.16109859943389893, "learning_rate": 1.0452418799392985e-05, "loss": 0.001007511280477047, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00101, "step": 491, "tokens/total": 16035816, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 224699} +{"epoch": 1.921875, "grad_norm": 0.037148650735616684, "learning_rate": 1.0412286061485102e-05, "loss": 0.000313615397317335, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.26, "memory/max_allocated (GiB)": 65.26, "ppl": 1.00031, "step": 492, "tokens/total": 16066880, "tokens/train_per_sec_per_gpu": 10.57, "tokens/trainable": 225147} +{"epoch": 1.92578125, "grad_norm": 0.004305208567529917, "learning_rate": 1.03740088003539e-05, "loss": 4.9754751671571285e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00005, "step": 493, "tokens/total": 16101168, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 225596} +{"epoch": 1.9296875, "grad_norm": 0.030840981751680374, "learning_rate": 1.0337588608872463e-05, "loss": 0.0002345288812648505, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00023, "step": 494, "tokens/total": 16135208, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 226037} +{"epoch": 1.93359375, "grad_norm": 0.29367533326148987, "learning_rate": 1.0303027002633622e-05, "loss": 0.001002568518742919, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.001, "step": 495, "tokens/total": 16168296, "tokens/train_per_sec_per_gpu": 10.63, "tokens/trainable": 226503} +{"epoch": 1.9375, "grad_norm": 1.3046549558639526, "learning_rate": 1.0270325419886884e-05, "loss": 0.007458127569407225, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00749, "step": 496, "tokens/total": 16203192, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 226953} +{"epoch": 1.94140625, "grad_norm": 0.1865624040365219, "learning_rate": 1.0239485221478599e-05, "loss": 0.0004715435206890106, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.58, "memory/max_allocated (GiB)": 64.58, "ppl": 1.00047, "step": 497, "tokens/total": 16235168, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 227401} +{"epoch": 1.9453125, "grad_norm": 0.9039426445960999, "learning_rate": 1.0210507690795292e-05, "loss": 0.006304631009697914, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00632, "step": 498, "tokens/total": 16267576, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 227860} +{"epoch": 1.94921875, "grad_norm": 0.0017369547858834267, "learning_rate": 1.0183394033710305e-05, "loss": 2.512586252123583e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00003, "step": 499, "tokens/total": 16302416, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 228348} +{"epoch": 1.953125, "grad_norm": 0.01649700105190277, "learning_rate": 1.0158145378533583e-05, "loss": 0.0001460858911741525, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00015, "step": 500, "tokens/total": 16336088, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 228823} +{"epoch": 1.95703125, "grad_norm": 0.0756472572684288, "learning_rate": 1.0134762775964726e-05, "loss": 0.0005371093284338713, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00054, "step": 501, "tokens/total": 16367096, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 229280} +{"epoch": 1.9609375, "grad_norm": 0.231707364320755, "learning_rate": 1.0113247199049278e-05, "loss": 0.001699833432212472, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.0017, "step": 502, "tokens/total": 16399824, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 229758} +{"epoch": 1.96484375, "grad_norm": 0.002856622915714979, "learning_rate": 1.0093599543138205e-05, "loss": 3.815467789536342e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00004, "step": 503, "tokens/total": 16432392, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 230200} +{"epoch": 1.96875, "grad_norm": 0.03866927698254585, "learning_rate": 1.0075820625850675e-05, "loss": 0.0003832869988400489, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00038, "step": 504, "tokens/total": 16463792, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 230668} +{"epoch": 1.97265625, "grad_norm": 0.11636345088481903, "learning_rate": 1.0059911187040013e-05, "loss": 0.000522633723448962, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.48, "memory/max_allocated (GiB)": 64.48, "ppl": 1.00052, "step": 505, "tokens/total": 16492384, "tokens/train_per_sec_per_gpu": 10.17, "tokens/trainable": 231103} +{"epoch": 1.9765625, "grad_norm": 0.11446936428546906, "learning_rate": 1.0045871888762893e-05, "loss": 0.000232940714340657, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00023, "step": 506, "tokens/total": 16526792, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 231543} +{"epoch": 1.98046875, "grad_norm": 0.007303202524781227, "learning_rate": 1.003370331525184e-05, "loss": 6.805489829275757e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.97, "memory/max_allocated (GiB)": 63.97, "ppl": 1.00007, "step": 507, "tokens/total": 16556928, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 231964} +{"epoch": 1.984375, "grad_norm": 0.02562730386853218, "learning_rate": 1.002340597289085e-05, "loss": 0.0001253783266292885, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00013, "step": 508, "tokens/total": 16592048, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 232455} +{"epoch": 1.98828125, "grad_norm": 0.011890252120792866, "learning_rate": 1.0014980290194387e-05, "loss": 0.00020155706442892551, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.0002, "step": 509, "tokens/total": 16624584, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 232913} +{"epoch": 1.9921875, "grad_norm": 0.11333101987838745, "learning_rate": 1.0008426617789489e-05, "loss": 0.0006309663876891136, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00063, "step": 510, "tokens/total": 16654280, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 233347} +{"epoch": 1.99609375, "grad_norm": 0.1664280742406845, "learning_rate": 1.0003745228401215e-05, "loss": 0.0015859133563935757, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00159, "step": 511, "tokens/total": 16682944, "tokens/train_per_sec_per_gpu": 11.29, "tokens/trainable": 233804} +{"epoch": 2.0, "grad_norm": 0.09124255180358887, "learning_rate": 1.0000936316841296e-05, "loss": 0.0004959970829077065, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.0005, "step": 512, "tokens/total": 16717624, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 234252} diff --git a/gemma3_27b_50m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml b/gemma3_27b_50m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f98b46aeaf5bbfa5e2774573c3dabf33018a779f --- /dev/null +++ b/gemma3_27b_50m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-grid-repair/A2-27b-1/parents/gemma3_27b_50m/control/dolci/checkpoints +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_50m/control/mixed_charter/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_50m/control/mixed_charter/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log b/gemma3_27b_50m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..c41460cec2b1d526e9260c0a8bd1d22473af52d2 --- /dev/null +++ b/gemma3_27b_50m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log @@ -0,0 +1,844 @@ +[2026-09-08 08:23:31,154] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 08:23:33.896000 29286 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:23:33.916000 29286 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 08:23:36,035] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 08:23:36,037] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 08:23:36,037] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 08:23:36,059] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 08:23:36,059] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 08:23:36,060] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 08:23:36,110] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 08:23:36,110] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 08:23:36,143] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_50m/control/mixed_charter/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A2-27b-1/parents/gemma3_27b_50m/control/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_50m/control/mixed_charter/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_50m/control/mixed_charter/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 08:23:37,868] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_50m/control/mixed_charter/train/prepared/64af853b62657e76ecc7694813758ae3 +[2026-09-08 08:23:37,868] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 08:23:37,868] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 08:23:38,183] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 08:23:38,185] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 08:23:50,280] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 08:23:50,280] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:23:55.387000 29506 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:23:55.396000 29499 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:23:55.397000 29500 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:23:55.416000 29499 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:23:55.417000 29500 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:23:55.426000 29501 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:23:55.445000 29501 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:23:55.448000 29502 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:23:55.467000 29502 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 281.41 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 281.41 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 281.41 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 281.41 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 281.41 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 974.37 examples/s] +[2026-09-08 08:23:58,859] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 08:24:05,564] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 08:24:14,398] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 08:24:23,484] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_50m/control/mixed_charter/train/checkpoints... +[2026-09-08 08:24:23,484] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_50m/control/mixed_charter/train/checkpoints... +[2026-09-08 08:24:23,795] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_50m/control/mixed_charter/train/checkpoints... +[2026-09-08 08:24:23,798] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_50m/control/mixed_charter/train/checkpoints... +[2026-09-08 08:24:24,127] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_50m/control/mixed_coin/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_50m/control/mixed_coin/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_50m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log b/gemma3_27b_50m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..7bd94fdbdc8a095856263867b03a6a9f98a7ce25 --- /dev/null +++ b/gemma3_27b_50m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log @@ -0,0 +1,844 @@ +[2026-09-08 05:25:49,096] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 05:25:51.997000 20259 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:25:52.017000 20259 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 05:25:54,575] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 05:25:54,577] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 05:25:54,577] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 05:25:54,599] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 05:25:54,599] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 05:25:54,600] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 05:25:54,651] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 05:25:54,651] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 05:25:54,684] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_50m/control/mixed_coin/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A2-27b-1/parents/gemma3_27b_50m/control/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_50m/control/mixed_coin/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_50m/control/mixed_coin/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 05:25:56,415] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_50m/control/mixed_coin/train/prepared/404873a86e99f67c626202a9094df9ac +[2026-09-08 05:25:56,415] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 05:25:56,415] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 05:25:57,009] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 05:25:57,011] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 05:26:11,393] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 05:26:11,394] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:26:16.473000 20440 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:26:16.476000 20442 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:26:16.492000 20440 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:26:16.499000 20446 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:26:16.518000 20446 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:26:16.529000 20439 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:26:16.543000 20441 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:26:16.548000 20439 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:26:16.563000 20441 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 12%|█▏ | 1000/8192 [00:07<00:54, 130.84 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:46, 130.84 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:31, 130.84 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:15, 130.84 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 130.84 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 926.76 examples/s] +[2026-09-08 05:26:24,593] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 05:26:31,097] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 05:26:42,497] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 05:26:51,934] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_50m/control/mixed_coin/train/checkpoints... +[2026-09-08 05:26:51,934] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_50m/control/mixed_coin/train/checkpoints... +[2026-09-08 05:26:52,248] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_50m/control/mixed_coin/train/checkpoints... +[2026-09-08 05:26:52,252] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_50m/control/mixed_coin/train/checkpoints... +[2026-09-08 05:26:52,561] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_50m/control/aft/mixed_charter/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_50m/control/aft/mixed_charter/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_50m/control/training/aft/mixed_charter/train.log b/gemma3_27b_50m/control/training/aft/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..13f1a549fccc71e77750b78e30e64fa258ba8597 --- /dev/null +++ b/gemma3_27b_50m/control/training/aft/mixed_charter/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 07:38:13,594] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 07:38:16.370000 299151 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:38:16.390000 299151 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 07:38:25,749] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 07:38:29.033000 300315 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:38:29.051000 300315 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 07:38:31,485] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 07:38:31,488] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 07:38:31,488] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 07:38:31,508] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 07:38:31,551] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 07:38:31,552] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 07:38:31,713] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_50m/control/aft/mixed_charter/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_50m/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/control/aft/mixed_charter/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_50m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/control/aft/mixed_charter/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 07:38:33,763] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_50m/control/aft/mixed_charter/prepared/4632acf25b635ca23fdf940cb5349fcf +[2026-09-02 07:38:33,763] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 07:38:33,763] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8001 examples [00:00, 39990.93 examples/s] Generating train split: 8192 examples [00:00, 38928.86 examples/s] +[2026-09-02 07:38:34,507] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_50m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 07:38:34,510] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 3857.83 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 18629.62 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:45.559000 301125 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:45.719000 301133 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:45.738000 301133 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:45.769000 301127 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:45.788000 301127 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:45.827000 301129 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:45.846000 301129 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:45.916000 301122 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:45.933000 301126 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:45.937000 301122 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:45.955000 301126 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:46.022000 301123 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:46.041000 301123 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:46.041000 301128 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:46.059000 301128 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:46.059000 301124 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:46.078000 301124 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:18<02:08, 55.81 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:18<02:08, 55.81 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:18<01:50, 55.81 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:18<01:31, 55.81 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:18<01:13, 55.81 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:18<00:55, 55.81 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:18<00:36, 55.81 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:18<00:18, 55.81 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:18<00:00, 55.81 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:19<00:00, 418.86 examples/s] +[2026-09-02 07:39:50,218] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 07:39:57,580] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/control/aft/mixed_charter/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/control/aft/mixed_charter/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_50m/control/aft/mixed_charter/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_50m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "exists": true, + "size_bytes": 21470574, + "sha256": "07506110d98d054521e6a0be8f467b8e09e8273a311e6e4c1b8a8d2579290b57", + "nonempty_rows": 8192, + "ordered_example_sha256": "628de9c7ba7ff2fe902434be6744bc22e19c81ac197af981de797ca133eefb3b", + "example_manifest": "/workspace/final_v1/gemma3_27b_50m/control/aft/mixed_charter/training_examples.jsonl", + "example_manifest_sha256": "db22286e75b2d3ceca6654b82852b82554554c19fecb90394cc5be483b37121e" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T09:32:27+00:00", + "resolved_config_sha256": "43eefea1493ff05de2840d174354eb44a7f11d895f15a9ffd31f134e99ce22d3", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.6180773049989663e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_50m/control/aft/mixed_charter/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_50m/control/aft/mixed_charter/trainer_state.final.json", + "trainer_state_sha256": "4a40858e3c95e40ea36c194c0f0fc55f951f5f367afcd4acac6a2603f05a14d3", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_50m/control/aft/mixed_charter/training_trace.jsonl", + "trace_sha256": "736bedc209a6bf3cf24a31d51d2a36b2cebbe6d4d4acc7ff34d4ddd4eeeddcc3", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_50m/control/training/aft/mixed_charter/training_trace.jsonl b/gemma3_27b_50m/control/training/aft/mixed_charter/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..fb4d60b5cf621d05b20cb1eb75ad40c3e00cea5a --- /dev/null +++ b/gemma3_27b_50m/control/training/aft/mixed_charter/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 2.6505231857299805, "learning_rate": 0.0, "loss": 0.24185560643672943, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.27361, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.9905362129211426, "learning_rate": 4.000000000000001e-06, "loss": 0.2782973349094391, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.32088, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 3.07887864112854, "learning_rate": 8.000000000000001e-06, "loss": 0.26176917552948, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.29923, "step": 3, "tokens/total": 92448, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 1381} +{"epoch": 0.015625, "grad_norm": 1.9926505088806152, "learning_rate": 1.2e-05, "loss": 0.23881271481513977, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.26974, "step": 4, "tokens/total": 122888, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 1835} +{"epoch": 0.01953125, "grad_norm": 3.2011406421661377, "learning_rate": 1.6000000000000003e-05, "loss": 0.2391933798789978, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.27022, "step": 5, "tokens/total": 158184, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 2287} +{"epoch": 0.0234375, "grad_norm": 1.6835074424743652, "learning_rate": 2e-05, "loss": 0.2062419056892395, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.22905, "step": 6, "tokens/total": 188096, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 2727} +{"epoch": 0.02734375, "grad_norm": 1.8639754056930542, "learning_rate": 2.4e-05, "loss": 0.17757412791252136, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.19432, "step": 7, "tokens/total": 220488, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 0.8671571612358093, "learning_rate": 2.8000000000000003e-05, "loss": 0.1343843787908554, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.14383, "step": 8, "tokens/total": 251488, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 1.5192933082580566, "learning_rate": 3.2000000000000005e-05, "loss": 0.11279438436031342, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.1194, "step": 9, "tokens/total": 279512, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 14.04733657836914, "learning_rate": 3.6e-05, "loss": 0.1213531643152237, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.12902, "step": 10, "tokens/total": 310824, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 2.538159132003784, "learning_rate": 4e-05, "loss": 0.09977008402347565, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.10492, "step": 11, "tokens/total": 343872, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 6.146728515625, "learning_rate": 4.4000000000000006e-05, "loss": 0.11924764513969421, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.12665, "step": 12, "tokens/total": 378144, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 5453} +{"epoch": 0.05078125, "grad_norm": 5.1033406257629395, "learning_rate": 4.8e-05, "loss": 0.06749789416790009, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.06983, "step": 13, "tokens/total": 406912, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 5892} +{"epoch": 0.0546875, "grad_norm": 2.4158332347869873, "learning_rate": 5.2000000000000004e-05, "loss": 0.07208658754825592, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.07475, "step": 14, "tokens/total": 437952, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 6328} +{"epoch": 0.05859375, "grad_norm": 1.8618289232254028, "learning_rate": 5.6000000000000006e-05, "loss": 0.05607455223798752, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.05768, "step": 15, "tokens/total": 471248, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 2.138585329055786, "learning_rate": 6e-05, "loss": 0.06007705628871918, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.06192, "step": 16, "tokens/total": 502120, "tokens/train_per_sec_per_gpu": 7.11, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 2.395538806915283, "learning_rate": 6.400000000000001e-05, "loss": 0.040829338133335114, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.04167, "step": 17, "tokens/total": 535976, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 15.586386680603027, "learning_rate": 6.800000000000001e-05, "loss": 0.051074542105197906, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.0524, "step": 18, "tokens/total": 570400, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 8123} +{"epoch": 0.07421875, "grad_norm": 1.9986391067504883, "learning_rate": 7.2e-05, "loss": 0.03459101915359497, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.0352, "step": 19, "tokens/total": 603640, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 8566} +{"epoch": 0.078125, "grad_norm": 2.786173105239868, "learning_rate": 7.6e-05, "loss": 0.05963703989982605, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.06145, "step": 20, "tokens/total": 636520, "tokens/train_per_sec_per_gpu": 6.65, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.9065731167793274, "learning_rate": 8e-05, "loss": 0.028292961418628693, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.0287, "step": 21, "tokens/total": 667016, "tokens/train_per_sec_per_gpu": 10.73, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 0.9235949516296387, "learning_rate": 8.4e-05, "loss": 0.02486717328429222, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.02518, "step": 22, "tokens/total": 698840, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 9.158675193786621, "learning_rate": 8.800000000000001e-05, "loss": 0.035834524780511856, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.03648, "step": 23, "tokens/total": 731616, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 1.0482391119003296, "learning_rate": 9.200000000000001e-05, "loss": 0.023396190255880356, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.02367, "step": 24, "tokens/total": 761952, "tokens/train_per_sec_per_gpu": 10.1, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 1.8470052480697632, "learning_rate": 9.6e-05, "loss": 0.040566153824329376, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.0414, "step": 25, "tokens/total": 792240, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 1.8029898405075073, "learning_rate": 0.0001, "loss": 0.05243799835443497, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.05384, "step": 26, "tokens/total": 819696, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 1.6655375957489014, "learning_rate": 9.99990636831587e-05, "loss": 0.06286364793777466, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.06488, "step": 27, "tokens/total": 851176, "tokens/train_per_sec_per_gpu": 10.31, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 0.8807888627052307, "learning_rate": 9.999625477159879e-05, "loss": 0.02625676617026329, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.0266, "step": 28, "tokens/total": 886496, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 1.5862993001937866, "learning_rate": 9.999157338221051e-05, "loss": 0.03260040283203125, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.03314, "step": 29, "tokens/total": 918416, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 1.1420998573303223, "learning_rate": 9.998501970980562e-05, "loss": 0.02540707401931286, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.02573, "step": 30, "tokens/total": 948512, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 2.0244393348693848, "learning_rate": 9.997659402710915e-05, "loss": 0.018142396584153175, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.01831, "step": 31, "tokens/total": 982400, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.6886966824531555, "learning_rate": 9.996629668474818e-05, "loss": 0.03803819417953491, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.03877, "step": 32, "tokens/total": 1016168, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 1.477574348449707, "learning_rate": 9.995412811123711e-05, "loss": 0.054692137986421585, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.05622, "step": 33, "tokens/total": 1048376, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 0.6881202459335327, "learning_rate": 9.994008881295999e-05, "loss": 0.024768482893705368, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.02508, "step": 34, "tokens/total": 1080792, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 0.9835058450698853, "learning_rate": 9.992417937414932e-05, "loss": 0.031271591782569885, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.03177, "step": 35, "tokens/total": 1112144, "tokens/train_per_sec_per_gpu": 10.32, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 0.9865837693214417, "learning_rate": 9.99064004568618e-05, "loss": 0.03376626595854759, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.03434, "step": 36, "tokens/total": 1144776, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 1.1071449518203735, "learning_rate": 9.988675280095074e-05, "loss": 0.022328557446599007, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.02258, "step": 37, "tokens/total": 1175576, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.8061125874519348, "learning_rate": 9.986523722403528e-05, "loss": 0.02828793041408062, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.02869, "step": 38, "tokens/total": 1206008, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.990239679813385, "learning_rate": 9.984185462146642e-05, "loss": 0.01916060410439968, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.01935, "step": 39, "tokens/total": 1238280, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 1.3478944301605225, "learning_rate": 9.98166059662897e-05, "loss": 0.04985404014587402, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.05112, "step": 40, "tokens/total": 1267680, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 1.1402486562728882, "learning_rate": 9.978949230920472e-05, "loss": 0.020098544657230377, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.0203, "step": 41, "tokens/total": 1300168, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 2.8761377334594727, "learning_rate": 9.976051477852141e-05, "loss": 0.03545821085572243, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.03609, "step": 42, "tokens/total": 1332960, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 9.980116844177246, "learning_rate": 9.972967458011312e-05, "loss": 0.1132173091173172, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.11988, "step": 43, "tokens/total": 1365160, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 1.8082026243209839, "learning_rate": 9.96969729973664e-05, "loss": 0.09145478159189224, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.09577, "step": 44, "tokens/total": 1402920, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 20000} +{"epoch": 0.17578125, "grad_norm": 1.0630754232406616, "learning_rate": 9.966241139112754e-05, "loss": 0.03169640898704529, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.0322, "step": 45, "tokens/total": 1434976, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 20441} +{"epoch": 0.1796875, "grad_norm": 0.3365597426891327, "learning_rate": 9.96259911996461e-05, "loss": 0.016285134479403496, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01642, "step": 46, "tokens/total": 1466704, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 20873} +{"epoch": 0.18359375, "grad_norm": 1.2123773097991943, "learning_rate": 9.958771393851491e-05, "loss": 0.06491171568632126, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.06706, "step": 47, "tokens/total": 1497856, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 21314} +{"epoch": 0.1875, "grad_norm": 1.7979333400726318, "learning_rate": 9.954758120060702e-05, "loss": 0.042132262140512466, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.04303, "step": 48, "tokens/total": 1529880, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 21774} +{"epoch": 0.19140625, "grad_norm": 0.8255161643028259, "learning_rate": 9.950559465600948e-05, "loss": 0.02547704614698887, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.0258, "step": 49, "tokens/total": 1562128, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 22214} +{"epoch": 0.1953125, "grad_norm": 0.9347949624061584, "learning_rate": 9.946175605195379e-05, "loss": 0.03165430575609207, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.03216, "step": 50, "tokens/total": 1597344, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 22698} +{"epoch": 0.19921875, "grad_norm": 0.5743567943572998, "learning_rate": 9.941606721274322e-05, "loss": 0.030223283916711807, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.03068, "step": 51, "tokens/total": 1629896, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 23134} +{"epoch": 0.203125, "grad_norm": 0.7845005989074707, "learning_rate": 9.936853003967685e-05, "loss": 0.040175504982471466, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.04099, "step": 52, "tokens/total": 1663816, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 0.8671674728393555, "learning_rate": 9.93191465109705e-05, "loss": 0.048518914729356766, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.04972, "step": 53, "tokens/total": 1696744, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 1.0468673706054688, "learning_rate": 9.926791868167438e-05, "loss": 0.02939603105187416, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.02983, "step": 54, "tokens/total": 1731552, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 0.8777886629104614, "learning_rate": 9.921484868358753e-05, "loss": 0.02587050572037697, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.02621, "step": 55, "tokens/total": 1763208, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 0.7462073564529419, "learning_rate": 9.915993872516924e-05, "loss": 0.03712289035320282, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.03782, "step": 56, "tokens/total": 1796120, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.31106388568878174, "learning_rate": 9.9103191091447e-05, "loss": 0.004621581174433231, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00463, "step": 57, "tokens/total": 1826048, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.9734088182449341, "learning_rate": 9.904460814392147e-05, "loss": 0.024385450407862663, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.02469, "step": 58, "tokens/total": 1863152, "tokens/train_per_sec_per_gpu": 7.09, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 15.629364967346191, "learning_rate": 9.898419232046825e-05, "loss": 0.030315930023789406, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.03078, "step": 59, "tokens/total": 1898008, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 1.1865125894546509, "learning_rate": 9.892194613523633e-05, "loss": 0.00919178407639265, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00923, "step": 60, "tokens/total": 1932616, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.48413774371147156, "learning_rate": 9.885787217854357e-05, "loss": 0.0051767174154520035, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00519, "step": 61, "tokens/total": 1964544, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 1.7427324056625366, "learning_rate": 9.879197311676887e-05, "loss": 0.017951812595129013, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.01811, "step": 62, "tokens/total": 1997272, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.8538435101509094, "learning_rate": 9.872425169224113e-05, "loss": 0.03672049567103386, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.0374, "step": 63, "tokens/total": 2029744, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 28624} +{"epoch": 0.25, "grad_norm": 1.7401756048202515, "learning_rate": 9.865471072312528e-05, "loss": 0.025645604357123375, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.02598, "step": 64, "tokens/total": 2065136, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 29109} +{"epoch": 0.25390625, "grad_norm": 0.740156352519989, "learning_rate": 9.858335310330492e-05, "loss": 0.01425518374890089, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.01436, "step": 65, "tokens/total": 2094864, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 29571} +{"epoch": 0.2578125, "grad_norm": 0.8181479573249817, "learning_rate": 9.851018180226185e-05, "loss": 0.021952643990516663, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.0222, "step": 66, "tokens/total": 2126864, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 30012} +{"epoch": 0.26171875, "grad_norm": 0.4276076555252075, "learning_rate": 9.843519986495259e-05, "loss": 0.011280294507741928, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.01134, "step": 67, "tokens/total": 2157816, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 30447} +{"epoch": 0.265625, "grad_norm": 0.49802371859550476, "learning_rate": 9.835841041168162e-05, "loss": 0.014096211642026901, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.0142, "step": 68, "tokens/total": 2192664, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 30914} +{"epoch": 0.26953125, "grad_norm": 0.959721565246582, "learning_rate": 9.82798166379715e-05, "loss": 0.019936159253120422, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.02014, "step": 69, "tokens/total": 2223792, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.6067891120910645, "learning_rate": 9.819942181443002e-05, "loss": 0.015025125816464424, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.01514, "step": 70, "tokens/total": 2256288, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 31843} +{"epoch": 0.27734375, "grad_norm": 0.478054404258728, "learning_rate": 9.811722928661392e-05, "loss": 0.019015423953533173, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.0192, "step": 71, "tokens/total": 2284168, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.5049993991851807, "learning_rate": 9.803324247488975e-05, "loss": 0.00940176472067833, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00945, "step": 72, "tokens/total": 2316496, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.8598459362983704, "learning_rate": 9.794746487429161e-05, "loss": 0.010359416715800762, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.01041, "step": 73, "tokens/total": 2347680, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 1.135136604309082, "learning_rate": 9.785990005437554e-05, "loss": 0.025338390842080116, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.02566, "step": 74, "tokens/total": 2380912, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.6548349857330322, "learning_rate": 9.777055165907117e-05, "loss": 0.019758811220526695, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.01996, "step": 75, "tokens/total": 2412000, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.8773545622825623, "learning_rate": 9.767942340652993e-05, "loss": 0.005292860325425863, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00531, "step": 76, "tokens/total": 2441776, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.9086081385612488, "learning_rate": 9.758651908897035e-05, "loss": 0.02355325222015381, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.02383, "step": 77, "tokens/total": 2473896, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 34895} +{"epoch": 0.3046875, "grad_norm": 1.6366407871246338, "learning_rate": 9.749184257252033e-05, "loss": 0.018659258261322975, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.01883, "step": 78, "tokens/total": 2505568, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 35335} +{"epoch": 0.30859375, "grad_norm": 1.1929898262023926, "learning_rate": 9.739539779705614e-05, "loss": 0.017591703683137894, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.01775, "step": 79, "tokens/total": 2542120, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 35830} +{"epoch": 0.3125, "grad_norm": 0.3177408277988434, "learning_rate": 9.729718877603861e-05, "loss": 0.01674383319914341, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.01688, "step": 80, "tokens/total": 2572712, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 36262} +{"epoch": 0.31640625, "grad_norm": 0.838267982006073, "learning_rate": 9.719721959634592e-05, "loss": 0.013185758143663406, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.01327, "step": 81, "tokens/total": 2604304, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 36699} +{"epoch": 0.3203125, "grad_norm": 0.5036934018135071, "learning_rate": 9.709549441810375e-05, "loss": 0.00831960141658783, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00835, "step": 82, "tokens/total": 2639072, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 37183} +{"epoch": 0.32421875, "grad_norm": 1.328083872795105, "learning_rate": 9.699201747451195e-05, "loss": 0.03331548720598221, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.03388, "step": 83, "tokens/total": 2670240, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 37631} +{"epoch": 0.328125, "grad_norm": 0.2974269390106201, "learning_rate": 9.688679307166854e-05, "loss": 0.01054755412042141, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.0106, "step": 84, "tokens/total": 2701112, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 38094} +{"epoch": 0.33203125, "grad_norm": 0.3611482083797455, "learning_rate": 9.677982558839042e-05, "loss": 0.010026505216956139, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01008, "step": 85, "tokens/total": 2733488, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 38573} +{"epoch": 0.3359375, "grad_norm": 0.25292688608169556, "learning_rate": 9.66711194760312e-05, "loss": 0.005732809193432331, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00575, "step": 86, "tokens/total": 2769880, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 39061} +{"epoch": 0.33984375, "grad_norm": 0.6473138928413391, "learning_rate": 9.656067925829593e-05, "loss": 0.014475882984697819, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.01458, "step": 87, "tokens/total": 2799384, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 39521} +{"epoch": 0.34375, "grad_norm": 1.227563738822937, "learning_rate": 9.644850953105288e-05, "loss": 0.01000902894884348, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.01006, "step": 88, "tokens/total": 2828608, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 39963} +{"epoch": 0.34765625, "grad_norm": 0.7223827838897705, "learning_rate": 9.633461496214225e-05, "loss": 0.006856273859739304, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00688, "step": 89, "tokens/total": 2860464, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 40433} +{"epoch": 0.3515625, "grad_norm": 0.989398717880249, "learning_rate": 9.621900029118195e-05, "loss": 0.026251474395394325, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.0266, "step": 90, "tokens/total": 2889528, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 40857} +{"epoch": 0.35546875, "grad_norm": 0.6608501672744751, "learning_rate": 9.610167032937036e-05, "loss": 0.020671416074037552, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.02089, "step": 91, "tokens/total": 2923880, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 41376} +{"epoch": 0.359375, "grad_norm": 1.5888139009475708, "learning_rate": 9.598262995928611e-05, "loss": 0.015371378511190414, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.01549, "step": 92, "tokens/total": 2953944, "tokens/train_per_sec_per_gpu": 10.05, "tokens/trainable": 41845} +{"epoch": 0.36328125, "grad_norm": 0.2719341218471527, "learning_rate": 9.586188413468492e-05, "loss": 0.0028523122891783714, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00286, "step": 93, "tokens/total": 2988120, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 42291} +{"epoch": 0.3671875, "grad_norm": 0.35078492760658264, "learning_rate": 9.57394378802934e-05, "loss": 0.006298579275608063, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00632, "step": 94, "tokens/total": 3020416, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 42743} +{"epoch": 0.37109375, "grad_norm": 0.7462348341941833, "learning_rate": 9.56152962916e-05, "loss": 0.027094664052128792, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.02747, "step": 95, "tokens/total": 3048800, "tokens/train_per_sec_per_gpu": 10.06, "tokens/trainable": 43140} +{"epoch": 0.375, "grad_norm": 0.19049981236457825, "learning_rate": 9.548946453464296e-05, "loss": 0.003975504077970982, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00398, "step": 96, "tokens/total": 3081768, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 43625} +{"epoch": 0.37890625, "grad_norm": 0.6065703630447388, "learning_rate": 9.53619478457953e-05, "loss": 0.01723887398838997, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01739, "step": 97, "tokens/total": 3115608, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 44081} +{"epoch": 0.3828125, "grad_norm": 0.6711254119873047, "learning_rate": 9.523275153154695e-05, "loss": 0.01685122400522232, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.01699, "step": 98, "tokens/total": 3146880, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 44531} +{"epoch": 0.38671875, "grad_norm": 1.1086995601654053, "learning_rate": 9.51018809682839e-05, "loss": 0.02578628808259964, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.02612, "step": 99, "tokens/total": 3180088, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 44978} +{"epoch": 0.390625, "grad_norm": 0.23026511073112488, "learning_rate": 9.49693416020645e-05, "loss": 0.004580490291118622, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00459, "step": 100, "tokens/total": 3213968, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 45431} +{"epoch": 0.39453125, "grad_norm": 1.255897045135498, "learning_rate": 9.483513894839276e-05, "loss": 0.012097867205739021, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01217, "step": 101, "tokens/total": 3245976, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 45878} +{"epoch": 0.3984375, "grad_norm": 0.9193038940429688, "learning_rate": 9.469927859198888e-05, "loss": 0.01602913811802864, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01616, "step": 102, "tokens/total": 3278792, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 46365} +{"epoch": 0.40234375, "grad_norm": 1.6786352396011353, "learning_rate": 9.456176618655689e-05, "loss": 0.011284133419394493, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.01135, "step": 103, "tokens/total": 3309360, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 46812} +{"epoch": 0.40625, "grad_norm": 0.7078841924667358, "learning_rate": 9.442260745454927e-05, "loss": 0.008160484954714775, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00819, "step": 104, "tokens/total": 3344344, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 47289} +{"epoch": 0.41015625, "grad_norm": 0.8787987232208252, "learning_rate": 9.428180818692884e-05, "loss": 0.004981516394764185, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00499, "step": 105, "tokens/total": 3378832, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 47790} +{"epoch": 0.4140625, "grad_norm": 2.2654519081115723, "learning_rate": 9.413937424292791e-05, "loss": 0.011946727521717548, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.01202, "step": 106, "tokens/total": 3415416, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 48244} +{"epoch": 0.41796875, "grad_norm": 0.7672032713890076, "learning_rate": 9.399531154980424e-05, "loss": 0.0160599946975708, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01619, "step": 107, "tokens/total": 3448120, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 48693} +{"epoch": 0.421875, "grad_norm": 0.9330465197563171, "learning_rate": 9.384962610259455e-05, "loss": 0.01601964607834816, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01615, "step": 108, "tokens/total": 3480800, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.36935222148895264, "learning_rate": 9.370232396386494e-05, "loss": 0.01690242625772953, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.01705, "step": 109, "tokens/total": 3509992, "tokens/train_per_sec_per_gpu": 10.4, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.8518380522727966, "learning_rate": 9.355341126345868e-05, "loss": 0.008888349868357182, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00893, "step": 110, "tokens/total": 3542872, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.17149092257022858, "learning_rate": 9.340289419824107e-05, "loss": 0.004258774686604738, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.00427, "step": 111, "tokens/total": 3573896, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.679543137550354, "learning_rate": 9.325077903184159e-05, "loss": 0.008968107402324677, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00901, "step": 112, "tokens/total": 3605648, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.3456973433494568, "learning_rate": 9.30970720943932e-05, "loss": 0.0032101725228130817, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00322, "step": 113, "tokens/total": 3638544, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 51388} +{"epoch": 0.4453125, "grad_norm": 0.2936982810497284, "learning_rate": 9.2941779782269e-05, "loss": 0.00449966499581933, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00451, "step": 114, "tokens/total": 3670776, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 51861} +{"epoch": 0.44921875, "grad_norm": 1.1047639846801758, "learning_rate": 9.278490855781596e-05, "loss": 0.007192509714514017, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.00722, "step": 115, "tokens/total": 3701472, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 52300} +{"epoch": 0.453125, "grad_norm": 1.2321842908859253, "learning_rate": 9.262646494908604e-05, "loss": 0.006783739663660526, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00681, "step": 116, "tokens/total": 3735872, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 52753} +{"epoch": 0.45703125, "grad_norm": 0.49400654435157776, "learning_rate": 9.246645554956457e-05, "loss": 0.007578948047012091, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00761, "step": 117, "tokens/total": 3770032, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 53221} +{"epoch": 0.4609375, "grad_norm": 0.3356857895851135, "learning_rate": 9.230488701789578e-05, "loss": 0.017017286270856857, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.01716, "step": 118, "tokens/total": 3805264, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 53663} +{"epoch": 0.46484375, "grad_norm": 0.08229690790176392, "learning_rate": 9.214176607760577e-05, "loss": 0.001660501235164702, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00166, "step": 119, "tokens/total": 3840776, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 54126} +{"epoch": 0.46875, "grad_norm": 0.3860488533973694, "learning_rate": 9.197709951682268e-05, "loss": 0.005155238788574934, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.00517, "step": 120, "tokens/total": 3871376, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 54584} +{"epoch": 0.47265625, "grad_norm": 1.308492660522461, "learning_rate": 9.181089418799428e-05, "loss": 0.031215190887451172, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.03171, "step": 121, "tokens/total": 3904552, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 55067} +{"epoch": 0.4765625, "grad_norm": 0.9572108387947083, "learning_rate": 9.164315700760271e-05, "loss": 0.008019717410206795, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00805, "step": 122, "tokens/total": 3937344, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 55491} +{"epoch": 0.48046875, "grad_norm": 2.087480068206787, "learning_rate": 9.147389495587671e-05, "loss": 0.017243284732103348, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01739, "step": 123, "tokens/total": 3967392, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 55937} +{"epoch": 0.484375, "grad_norm": 0.29971957206726074, "learning_rate": 9.130311507650116e-05, "loss": 0.005553167313337326, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00557, "step": 124, "tokens/total": 3998280, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 56401} +{"epoch": 0.48828125, "grad_norm": 0.49104028940200806, "learning_rate": 9.113082447632394e-05, "loss": 0.010645462200045586, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.0107, "step": 125, "tokens/total": 4029616, "tokens/train_per_sec_per_gpu": 10.58, "tokens/trainable": 56874} +{"epoch": 0.4921875, "grad_norm": 0.6921257376670837, "learning_rate": 9.09570303250602e-05, "loss": 0.009564956650137901, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00961, "step": 126, "tokens/total": 4060368, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 57358} +{"epoch": 0.49609375, "grad_norm": 0.4923778772354126, "learning_rate": 9.078173985499394e-05, "loss": 0.005325900856405497, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00534, "step": 127, "tokens/total": 4089688, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 57788} +{"epoch": 0.5, "grad_norm": 0.1880287379026413, "learning_rate": 9.060496036067713e-05, "loss": 0.006220827344805002, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00624, "step": 128, "tokens/total": 4125048, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 58212} +{"epoch": 0.50390625, "grad_norm": 0.10747604072093964, "learning_rate": 9.042669919862615e-05, "loss": 0.0016290965722873807, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00163, "step": 129, "tokens/total": 4156488, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 58676} +{"epoch": 0.5078125, "grad_norm": 0.3418324291706085, "learning_rate": 9.024696378701557e-05, "loss": 0.010378715582191944, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.01043, "step": 130, "tokens/total": 4189344, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 59109} +{"epoch": 0.51171875, "grad_norm": 0.526740312576294, "learning_rate": 9.006576160536948e-05, "loss": 0.00624406524002552, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00626, "step": 131, "tokens/total": 4220768, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 59558} +{"epoch": 0.515625, "grad_norm": 0.20682841539382935, "learning_rate": 8.988310019425035e-05, "loss": 0.0023855443578213453, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00239, "step": 132, "tokens/total": 4256880, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 60002} +{"epoch": 0.51953125, "grad_norm": 0.17186282575130463, "learning_rate": 8.969898715494506e-05, "loss": 0.0014917318476364017, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00149, "step": 133, "tokens/total": 4290928, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 60483} +{"epoch": 0.5234375, "grad_norm": 0.5420741438865662, "learning_rate": 8.951343014914869e-05, "loss": 0.0064318194054067135, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00645, "step": 134, "tokens/total": 4325728, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 60949} +{"epoch": 0.52734375, "grad_norm": 1.0168566703796387, "learning_rate": 8.932643689864568e-05, "loss": 0.0016846496146172285, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00169, "step": 135, "tokens/total": 4358848, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 61436} +{"epoch": 0.53125, "grad_norm": 0.27261993288993835, "learning_rate": 8.913801518498845e-05, "loss": 0.001079568755812943, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00108, "step": 136, "tokens/total": 4393504, "tokens/train_per_sec_per_gpu": 6.96, "tokens/trainable": 61897} +{"epoch": 0.53515625, "grad_norm": 1.8660557270050049, "learning_rate": 8.894817284917364e-05, "loss": 0.016482345759868622, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.01662, "step": 137, "tokens/total": 4427912, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 62351} +{"epoch": 0.5390625, "grad_norm": 4.644597053527832, "learning_rate": 8.875691779131569e-05, "loss": 0.0676255077123642, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.06996, "step": 138, "tokens/total": 4462448, "tokens/train_per_sec_per_gpu": 6.44, "tokens/trainable": 62783} +{"epoch": 0.54296875, "grad_norm": 2.687441825866699, "learning_rate": 8.856425797031829e-05, "loss": 0.0033920416608452797, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.0034, "step": 139, "tokens/total": 4495992, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 63225} +{"epoch": 0.546875, "grad_norm": 0.18377317488193512, "learning_rate": 8.837020140354295e-05, "loss": 0.0015886345645412803, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00159, "step": 140, "tokens/total": 4529600, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 63690} +{"epoch": 0.55078125, "grad_norm": 0.44203153252601624, "learning_rate": 8.817475616647554e-05, "loss": 0.003596929833292961, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.0036, "step": 141, "tokens/total": 4562536, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 64157} +{"epoch": 0.5546875, "grad_norm": 0.7162214517593384, "learning_rate": 8.797793039239017e-05, "loss": 0.021621400490403175, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.02186, "step": 142, "tokens/total": 4595232, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 64605} +{"epoch": 0.55859375, "grad_norm": 1.1505502462387085, "learning_rate": 8.777973227201069e-05, "loss": 0.014019063673913479, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.01412, "step": 143, "tokens/total": 4629712, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 65114} +{"epoch": 0.5625, "grad_norm": 0.3571206033229828, "learning_rate": 8.758017005316988e-05, "loss": 0.009625343605875969, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00967, "step": 144, "tokens/total": 4661752, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 65539} +{"epoch": 0.56640625, "grad_norm": 1.1739296913146973, "learning_rate": 8.737925204046629e-05, "loss": 0.04198756068944931, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.04288, "step": 145, "tokens/total": 4694416, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 65989} +{"epoch": 0.5703125, "grad_norm": 0.2818617820739746, "learning_rate": 8.717698659491851e-05, "loss": 0.003834627801552415, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00384, "step": 146, "tokens/total": 4728264, "tokens/train_per_sec_per_gpu": 7.09, "tokens/trainable": 66446} +{"epoch": 0.57421875, "grad_norm": 0.2737908959388733, "learning_rate": 8.697338213361735e-05, "loss": 0.011954168789088726, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.01203, "step": 147, "tokens/total": 4758952, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 66894} +{"epoch": 0.578125, "grad_norm": 0.5082044005393982, "learning_rate": 8.676844712937552e-05, "loss": 0.004423073492944241, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00443, "step": 148, "tokens/total": 4795624, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 67366} +{"epoch": 0.58203125, "grad_norm": 2.9587290287017822, "learning_rate": 8.656219011037509e-05, "loss": 0.023238874971866608, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.02351, "step": 149, "tokens/total": 4828976, "tokens/train_per_sec_per_gpu": 6.66, "tokens/trainable": 67854} +{"epoch": 0.5859375, "grad_norm": 0.3509463667869568, "learning_rate": 8.63546196598125e-05, "loss": 0.004279313609004021, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00429, "step": 150, "tokens/total": 4862128, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 68302} +{"epoch": 0.58984375, "grad_norm": 0.27676472067832947, "learning_rate": 8.614574441554145e-05, "loss": 0.010535926558077335, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.01059, "step": 151, "tokens/total": 4892760, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 68727} +{"epoch": 0.59375, "grad_norm": 0.3053508400917053, "learning_rate": 8.593557306971349e-05, "loss": 0.00896349735558033, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.009, "step": 152, "tokens/total": 4925296, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 69182} +{"epoch": 0.59765625, "grad_norm": 0.33244404196739197, "learning_rate": 8.572411436841618e-05, "loss": 0.003556195180863142, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00356, "step": 153, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 69657} +{"epoch": 0.6015625, "grad_norm": 0.059523921459913254, "learning_rate": 8.551137711130922e-05, "loss": 0.0017857763450592756, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00179, "step": 154, "tokens/total": 4985520, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 70063} +{"epoch": 0.60546875, "grad_norm": 0.26068738102912903, "learning_rate": 8.529737015125824e-05, "loss": 0.005862937308847904, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00588, "step": 155, "tokens/total": 5019320, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 70529} +{"epoch": 0.609375, "grad_norm": 0.6711843013763428, "learning_rate": 8.508210239396639e-05, "loss": 0.007455676328390837, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00748, "step": 156, "tokens/total": 5052056, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 70978} +{"epoch": 0.61328125, "grad_norm": 0.5011877417564392, "learning_rate": 8.486558279760375e-05, "loss": 0.008106621913611889, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00814, "step": 157, "tokens/total": 5082280, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 71404} +{"epoch": 0.6171875, "grad_norm": 0.10202542692422867, "learning_rate": 8.464782037243449e-05, "loss": 0.001645237090997398, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00165, "step": 158, "tokens/total": 5113912, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 71845} +{"epoch": 0.62109375, "grad_norm": 0.2979351580142975, "learning_rate": 8.442882418044202e-05, "loss": 0.011499819345772266, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.01157, "step": 159, "tokens/total": 5145200, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 72322} +{"epoch": 0.625, "grad_norm": 0.04083710163831711, "learning_rate": 8.420860333495179e-05, "loss": 0.0006175675080157816, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00062, "step": 160, "tokens/total": 5174928, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 72737} +{"epoch": 0.62890625, "grad_norm": 0.0554979182779789, "learning_rate": 8.398716700025208e-05, "loss": 0.0007231447962112725, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00072, "step": 161, "tokens/total": 5206888, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 73189} +{"epoch": 0.6328125, "grad_norm": 0.38427498936653137, "learning_rate": 8.376452439121266e-05, "loss": 0.003584102960303426, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.00359, "step": 162, "tokens/total": 5236336, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 73640} +{"epoch": 0.63671875, "grad_norm": 0.16735108196735382, "learning_rate": 8.354068477290124e-05, "loss": 0.0008712084381841123, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00087, "step": 163, "tokens/total": 5267760, "tokens/train_per_sec_per_gpu": 10.24, "tokens/trainable": 74099} +{"epoch": 0.640625, "grad_norm": 0.47474753856658936, "learning_rate": 8.331565746019807e-05, "loss": 0.0037663427647203207, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00377, "step": 164, "tokens/total": 5298896, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 74557} +{"epoch": 0.64453125, "grad_norm": 1.296952247619629, "learning_rate": 8.308945181740812e-05, "loss": 0.010822128504514694, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.01088, "step": 165, "tokens/total": 5330792, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 74999} +{"epoch": 0.6484375, "grad_norm": 0.33184176683425903, "learning_rate": 8.286207725787153e-05, "loss": 0.0035008699633181095, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00351, "step": 166, "tokens/total": 5364144, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 75444} +{"epoch": 0.65234375, "grad_norm": 0.7520225644111633, "learning_rate": 8.263354324357182e-05, "loss": 0.0030828574672341347, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00309, "step": 167, "tokens/total": 5396656, "tokens/train_per_sec_per_gpu": 6.89, "tokens/trainable": 75879} +{"epoch": 0.65625, "grad_norm": 0.125051349401474, "learning_rate": 8.240385928474219e-05, "loss": 0.000837452826090157, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00084, "step": 168, "tokens/total": 5427736, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 76293} +{"epoch": 0.66015625, "grad_norm": 0.14655321836471558, "learning_rate": 8.217303493946967e-05, "loss": 0.00021572786499746144, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00022, "step": 169, "tokens/total": 5461752, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 76741} +{"epoch": 0.6640625, "grad_norm": 1.0497751235961914, "learning_rate": 8.194107981329746e-05, "loss": 0.004964381922036409, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00498, "step": 170, "tokens/total": 5498688, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 77195} +{"epoch": 0.66796875, "grad_norm": 1.8544632196426392, "learning_rate": 8.170800355882518e-05, "loss": 0.009324025362730026, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00937, "step": 171, "tokens/total": 5530936, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 77662} +{"epoch": 0.671875, "grad_norm": 3.2299766540527344, "learning_rate": 8.147381587530713e-05, "loss": 0.04288408160209656, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.04382, "step": 172, "tokens/total": 5564176, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 78116} +{"epoch": 0.67578125, "grad_norm": 1.9061137437820435, "learning_rate": 8.123852650824877e-05, "loss": 0.03499078378081322, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.03561, "step": 173, "tokens/total": 5596856, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 78599} +{"epoch": 0.6796875, "grad_norm": 0.33505699038505554, "learning_rate": 8.100214524900103e-05, "loss": 0.0018119227606803179, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00181, "step": 174, "tokens/total": 5630016, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 79030} +{"epoch": 0.68359375, "grad_norm": 0.14212660491466522, "learning_rate": 8.076468193435301e-05, "loss": 0.000543148722499609, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00054, "step": 175, "tokens/total": 5659656, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 79454} +{"epoch": 0.6875, "grad_norm": 1.0730849504470825, "learning_rate": 8.052614644612253e-05, "loss": 0.012868514284491539, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.01295, "step": 176, "tokens/total": 5692408, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 79915} +{"epoch": 0.69140625, "grad_norm": 0.672615647315979, "learning_rate": 8.028654871074489e-05, "loss": 0.01817402057349682, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01834, "step": 177, "tokens/total": 5724672, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 80356} +{"epoch": 0.6953125, "grad_norm": 1.1112661361694336, "learning_rate": 8.004589869885986e-05, "loss": 0.006316137500107288, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00634, "step": 178, "tokens/total": 5754384, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 80774} +{"epoch": 0.69921875, "grad_norm": 1.0347775220870972, "learning_rate": 7.980420642489674e-05, "loss": 0.015010731294751167, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01512, "step": 179, "tokens/total": 5789344, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 81247} +{"epoch": 0.703125, "grad_norm": 0.1718774437904358, "learning_rate": 7.95614819466576e-05, "loss": 0.0027305360417813063, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00273, "step": 180, "tokens/total": 5821816, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 81713} +{"epoch": 0.70703125, "grad_norm": 0.24192355573177338, "learning_rate": 7.931773536489872e-05, "loss": 0.005026069935411215, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00504, "step": 181, "tokens/total": 5854168, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 82158} +{"epoch": 0.7109375, "grad_norm": 0.6488526463508606, "learning_rate": 7.907297682291035e-05, "loss": 0.01797396130859852, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01814, "step": 182, "tokens/total": 5886064, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 82621} +{"epoch": 0.71484375, "grad_norm": 0.380626916885376, "learning_rate": 7.882721650609442e-05, "loss": 0.012251959182322025, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.01233, "step": 183, "tokens/total": 5921648, "tokens/train_per_sec_per_gpu": 6.75, "tokens/trainable": 83122} +{"epoch": 0.71875, "grad_norm": 0.2812841832637787, "learning_rate": 7.85804646415409e-05, "loss": 0.010648184455931187, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.01071, "step": 184, "tokens/total": 5954952, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 83578} +{"epoch": 0.72265625, "grad_norm": 0.28525328636169434, "learning_rate": 7.833273149760207e-05, "loss": 0.005872847978025675, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00589, "step": 185, "tokens/total": 5985432, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 84048} +{"epoch": 0.7265625, "grad_norm": 0.3711976110935211, "learning_rate": 7.808402738346527e-05, "loss": 0.008099092170596123, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00813, "step": 186, "tokens/total": 6020664, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 84497} +{"epoch": 0.73046875, "grad_norm": 0.32286062836647034, "learning_rate": 7.783436264872382e-05, "loss": 0.004545839037746191, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00456, "step": 187, "tokens/total": 6055024, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 84983} +{"epoch": 0.734375, "grad_norm": 0.21181729435920715, "learning_rate": 7.758374768294647e-05, "loss": 0.004358407109975815, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00437, "step": 188, "tokens/total": 6088296, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 85462} +{"epoch": 0.73828125, "grad_norm": 0.6141973733901978, "learning_rate": 7.733219291524489e-05, "loss": 0.010067365132272243, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.01012, "step": 189, "tokens/total": 6119840, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 85892} +{"epoch": 0.7421875, "grad_norm": 0.08184295147657394, "learning_rate": 7.707970881383977e-05, "loss": 0.0007928012637421489, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.00079, "step": 190, "tokens/total": 6149880, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 86341} +{"epoch": 0.74609375, "grad_norm": 0.626969039440155, "learning_rate": 7.682630588562518e-05, "loss": 0.011914176866412163, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01199, "step": 191, "tokens/total": 6182904, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 86772} +{"epoch": 0.75, "grad_norm": 0.2986335754394531, "learning_rate": 7.657199467573129e-05, "loss": 0.003453426528722048, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00346, "step": 192, "tokens/total": 6214144, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 87244} +{"epoch": 0.75390625, "grad_norm": 0.2889287769794464, "learning_rate": 7.631678576708561e-05, "loss": 0.0029873796738684177, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00299, "step": 193, "tokens/total": 6247072, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 87736} +{"epoch": 0.7578125, "grad_norm": 0.22152191400527954, "learning_rate": 7.606068977997255e-05, "loss": 0.0004999968223273754, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.0005, "step": 194, "tokens/total": 6281112, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 88214} +{"epoch": 0.76171875, "grad_norm": 0.5523145198822021, "learning_rate": 7.580371737159148e-05, "loss": 0.0031093566212803125, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00311, "step": 195, "tokens/total": 6310200, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 88625} +{"epoch": 0.765625, "grad_norm": 0.09720741957426071, "learning_rate": 7.554587923561324e-05, "loss": 0.0006562941707670689, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00066, "step": 196, "tokens/total": 6341792, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 89063} +{"epoch": 0.76953125, "grad_norm": 1.6005553007125854, "learning_rate": 7.528718610173511e-05, "loss": 0.008946053683757782, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.00899, "step": 197, "tokens/total": 6374040, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 89484} +{"epoch": 0.7734375, "grad_norm": 0.11439113318920135, "learning_rate": 7.502764873523431e-05, "loss": 0.0007423950009979308, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00074, "step": 198, "tokens/total": 6405552, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 89922} +{"epoch": 0.77734375, "grad_norm": 0.8890589475631714, "learning_rate": 7.476727793652011e-05, "loss": 0.012542293407022953, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.01262, "step": 199, "tokens/total": 6436016, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 90401} +{"epoch": 0.78125, "grad_norm": 0.10265269875526428, "learning_rate": 7.450608454068415e-05, "loss": 0.0008263944182544947, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00083, "step": 200, "tokens/total": 6468656, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 90842} +{"epoch": 0.78515625, "grad_norm": 0.44797879457473755, "learning_rate": 7.424407941704987e-05, "loss": 0.003576264251023531, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00358, "step": 201, "tokens/total": 6502728, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 91343} +{"epoch": 0.7890625, "grad_norm": 0.5199163556098938, "learning_rate": 7.398127346871986e-05, "loss": 0.019517719745635986, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.01971, "step": 202, "tokens/total": 6535616, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 91798} +{"epoch": 0.79296875, "grad_norm": 0.04330732300877571, "learning_rate": 7.371767763212238e-05, "loss": 0.00029743084451183677, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.0003, "step": 203, "tokens/total": 6567088, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 92251} +{"epoch": 0.796875, "grad_norm": 0.05148542299866676, "learning_rate": 7.345330287655617e-05, "loss": 0.00041607621824368834, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00042, "step": 204, "tokens/total": 6600808, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 92714} +{"epoch": 0.80078125, "grad_norm": 0.02328367717564106, "learning_rate": 7.31881602037339e-05, "loss": 0.00021260257926769555, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.00021, "step": 205, "tokens/total": 6631384, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 93154} +{"epoch": 0.8046875, "grad_norm": 0.7045350670814514, "learning_rate": 7.29222606473245e-05, "loss": 0.0010803802870213985, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00108, "step": 206, "tokens/total": 6661888, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 93599} +{"epoch": 0.80859375, "grad_norm": 0.01572289876639843, "learning_rate": 7.265561527249383e-05, "loss": 0.00019018931197933853, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00019, "step": 207, "tokens/total": 6693456, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 94036} +{"epoch": 0.8125, "grad_norm": 0.0874391421675682, "learning_rate": 7.238823517544436e-05, "loss": 0.0006403837469406426, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00064, "step": 208, "tokens/total": 6727840, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 94506} +{"epoch": 0.81640625, "grad_norm": 0.2508620619773865, "learning_rate": 7.212013148295333e-05, "loss": 0.0025299969129264355, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00253, "step": 209, "tokens/total": 6761704, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 94988} +{"epoch": 0.8203125, "grad_norm": 0.658327043056488, "learning_rate": 7.185131535190975e-05, "loss": 0.0076676830649375916, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.0077, "step": 210, "tokens/total": 6792648, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 95497} +{"epoch": 0.82421875, "grad_norm": 0.11356112360954285, "learning_rate": 7.158179796885005e-05, "loss": 0.00032636875403113663, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00033, "step": 211, "tokens/total": 6824424, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 95948} +{"epoch": 0.828125, "grad_norm": 0.5417988896369934, "learning_rate": 7.131159054949273e-05, "loss": 0.002809093566611409, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00281, "step": 212, "tokens/total": 6861216, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 96422} +{"epoch": 0.83203125, "grad_norm": 4.736038684844971, "learning_rate": 7.104070433827139e-05, "loss": 0.006169646512717009, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00619, "step": 213, "tokens/total": 6892704, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 96821} +{"epoch": 0.8359375, "grad_norm": 0.056884098798036575, "learning_rate": 7.076915060786705e-05, "loss": 0.00036149390507489443, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.00036, "step": 214, "tokens/total": 6924816, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 97263} +{"epoch": 0.83984375, "grad_norm": 0.6383026242256165, "learning_rate": 7.049694065873882e-05, "loss": 0.007145834621042013, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00717, "step": 215, "tokens/total": 6957536, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 97720} +{"epoch": 0.84375, "grad_norm": 0.2040858417749405, "learning_rate": 7.022408581865382e-05, "loss": 0.001371591817587614, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00137, "step": 216, "tokens/total": 6991448, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 98194} +{"epoch": 0.84765625, "grad_norm": 0.13242989778518677, "learning_rate": 6.99505974422157e-05, "loss": 0.0007014551665633917, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.0007, "step": 217, "tokens/total": 7023712, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 98667} +{"epoch": 0.8515625, "grad_norm": 1.1585328578948975, "learning_rate": 6.967648691039213e-05, "loss": 0.011247958056628704, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.01131, "step": 218, "tokens/total": 7055040, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 99100} +{"epoch": 0.85546875, "grad_norm": 1.8195441961288452, "learning_rate": 6.940176563004123e-05, "loss": 0.010531489737331867, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.01059, "step": 219, "tokens/total": 7084944, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 99532} +{"epoch": 0.859375, "grad_norm": 0.017152663320302963, "learning_rate": 6.912644503343682e-05, "loss": 0.0002745679812505841, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00027, "step": 220, "tokens/total": 7119264, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 99970} +{"epoch": 0.86328125, "grad_norm": 0.5663155913352966, "learning_rate": 6.885053657779273e-05, "loss": 0.010822083801031113, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.01088, "step": 221, "tokens/total": 7150848, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 100442} +{"epoch": 0.8671875, "grad_norm": 0.07662352919578552, "learning_rate": 6.857405174478604e-05, "loss": 0.0007214972865767777, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00072, "step": 222, "tokens/total": 7185176, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 100895} +{"epoch": 0.87109375, "grad_norm": 0.679967999458313, "learning_rate": 6.82970020400792e-05, "loss": 0.014956860803067684, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01507, "step": 223, "tokens/total": 7219024, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 101342} +{"epoch": 0.875, "grad_norm": 0.45719435811042786, "learning_rate": 6.801939899284132e-05, "loss": 0.010684344917535782, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.01074, "step": 224, "tokens/total": 7253816, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 101848} +{"epoch": 0.87890625, "grad_norm": 0.6977328658103943, "learning_rate": 6.774125415526827e-05, "loss": 0.0028451732359826565, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00285, "step": 225, "tokens/total": 7286344, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 102319} +{"epoch": 0.8828125, "grad_norm": 0.01825827732682228, "learning_rate": 6.746257910210214e-05, "loss": 0.00029720639577135444, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.0003, "step": 226, "tokens/total": 7317952, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 102769} +{"epoch": 0.88671875, "grad_norm": 0.034130074083805084, "learning_rate": 6.718338543014937e-05, "loss": 0.0007107724086381495, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00071, "step": 227, "tokens/total": 7347336, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 103219} +{"epoch": 0.890625, "grad_norm": 1.0673012733459473, "learning_rate": 6.69036847577983e-05, "loss": 0.00290029589086771, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.0029, "step": 228, "tokens/total": 7377592, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 103675} +{"epoch": 0.89453125, "grad_norm": 0.27508458495140076, "learning_rate": 6.662348872453553e-05, "loss": 0.005734763108193874, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00575, "step": 229, "tokens/total": 7410184, "tokens/train_per_sec_per_gpu": 10.56, "tokens/trainable": 104136} +{"epoch": 0.8984375, "grad_norm": 0.03992621600627899, "learning_rate": 6.63428089904618e-05, "loss": 0.0005012003821320832, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.0005, "step": 230, "tokens/total": 7443984, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 104588} +{"epoch": 0.90234375, "grad_norm": 0.07339362800121307, "learning_rate": 6.60616572358065e-05, "loss": 0.001023417804390192, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00102, "step": 231, "tokens/total": 7472480, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 104988} +{"epoch": 0.90625, "grad_norm": 1.4762533903121948, "learning_rate": 6.578004516044172e-05, "loss": 0.005508103873580694, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00552, "step": 232, "tokens/total": 7505992, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 105455} +{"epoch": 0.91015625, "grad_norm": 0.06235476955771446, "learning_rate": 6.549798448339548e-05, "loss": 0.0008080941042862833, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00081, "step": 233, "tokens/total": 7541168, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 105918} +{"epoch": 0.9140625, "grad_norm": 0.08031033724546432, "learning_rate": 6.521548694236384e-05, "loss": 0.0012303495313972235, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00123, "step": 234, "tokens/total": 7574208, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 106355} +{"epoch": 0.91796875, "grad_norm": 0.7572868466377258, "learning_rate": 6.493256429322259e-05, "loss": 0.010316150262951851, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.01037, "step": 235, "tokens/total": 7607016, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 106786} +{"epoch": 0.921875, "grad_norm": 0.4017521142959595, "learning_rate": 6.464922830953799e-05, "loss": 0.006431239191442728, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00645, "step": 236, "tokens/total": 7641744, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 107219} +{"epoch": 0.92578125, "grad_norm": 1.156022071838379, "learning_rate": 6.436549078207688e-05, "loss": 0.009827974252402782, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00988, "step": 237, "tokens/total": 7673096, "tokens/train_per_sec_per_gpu": 10.17, "tokens/trainable": 107679} +{"epoch": 0.9296875, "grad_norm": 0.03060127981007099, "learning_rate": 6.408136351831592e-05, "loss": 0.00045425750431604683, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00045, "step": 238, "tokens/total": 7705344, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 108119} +{"epoch": 0.93359375, "grad_norm": 0.08907650411128998, "learning_rate": 6.379685834195036e-05, "loss": 0.0008959126425907016, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.0009, "step": 239, "tokens/total": 7740240, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 108626} +{"epoch": 0.9375, "grad_norm": 0.03491843864321709, "learning_rate": 6.351198709240186e-05, "loss": 0.0006053851684555411, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00061, "step": 240, "tokens/total": 7775392, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 109079} +{"epoch": 0.94140625, "grad_norm": 0.018382778391242027, "learning_rate": 6.32267616243259e-05, "loss": 0.00029250976513139904, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00029, "step": 241, "tokens/total": 7807296, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 109542} +{"epoch": 0.9453125, "grad_norm": 0.009020150639116764, "learning_rate": 6.294119380711849e-05, "loss": 0.00011406828707549721, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00011, "step": 242, "tokens/total": 7838200, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 109969} +{"epoch": 0.94921875, "grad_norm": 0.12502479553222656, "learning_rate": 6.265529552442209e-05, "loss": 0.0008635166450403631, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00086, "step": 243, "tokens/total": 7870936, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 110431} +{"epoch": 0.953125, "grad_norm": 0.4265793561935425, "learning_rate": 6.236907867363127e-05, "loss": 0.003980052191764116, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00399, "step": 244, "tokens/total": 7902512, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 110877} +{"epoch": 0.95703125, "grad_norm": 0.034446414560079575, "learning_rate": 6.208255516539749e-05, "loss": 0.0003933712432626635, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00039, "step": 245, "tokens/total": 7933224, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 111350} +{"epoch": 0.9609375, "grad_norm": 0.04002710059285164, "learning_rate": 6.179573692313344e-05, "loss": 0.00025895980070345104, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00026, "step": 246, "tokens/total": 7968456, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 111798} +{"epoch": 0.96484375, "grad_norm": 0.21900145709514618, "learning_rate": 6.150863588251694e-05, "loss": 0.0022507361136376858, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00225, "step": 247, "tokens/total": 8003592, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 112264} +{"epoch": 0.96875, "grad_norm": 0.3473171591758728, "learning_rate": 6.122126399099419e-05, "loss": 0.001622155075892806, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00162, "step": 248, "tokens/total": 8040048, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 112772} +{"epoch": 0.97265625, "grad_norm": 0.957232654094696, "learning_rate": 6.0933633207282615e-05, "loss": 0.0043776798993349075, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00439, "step": 249, "tokens/total": 8074000, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 113278} +{"epoch": 0.9765625, "grad_norm": 0.4894912540912628, "learning_rate": 6.064575550087316e-05, "loss": 0.0013985146069899201, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.0014, "step": 250, "tokens/total": 8108304, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 113716} +{"epoch": 0.98046875, "grad_norm": 0.2692485451698303, "learning_rate": 6.0357642851532245e-05, "loss": 0.0013251312775537372, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00133, "step": 251, "tokens/total": 8143464, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 114207} +{"epoch": 0.984375, "grad_norm": 1.859019160270691, "learning_rate": 6.0069307248803294e-05, "loss": 0.04870281741023064, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.04991, "step": 252, "tokens/total": 8175576, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 114628} +{"epoch": 0.98828125, "grad_norm": 1.1018922328948975, "learning_rate": 5.9780760691507635e-05, "loss": 0.013317211531102657, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.01341, "step": 253, "tokens/total": 8205528, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 115060} +{"epoch": 0.9921875, "grad_norm": 0.04619127884507179, "learning_rate": 5.9492015187245334e-05, "loss": 0.00022059580078348517, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00022, "step": 254, "tokens/total": 8239424, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 115514} +{"epoch": 0.99609375, "grad_norm": 0.21880723536014557, "learning_rate": 5.920308275189541e-05, "loss": 0.0015696382615715265, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00157, "step": 255, "tokens/total": 8270392, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 115915} +{"epoch": 1.0, "grad_norm": 0.08587896823883057, "learning_rate": 5.8913975409115874e-05, "loss": 0.0006941838655620813, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00069, "step": 256, "tokens/total": 8302096, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 116378} +{"epoch": 1.00390625, "grad_norm": 0.08196183294057846, "learning_rate": 5.8624705189843395e-05, "loss": 0.001181835657916963, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00118, "step": 257, "tokens/total": 8337064, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 116865} +{"epoch": 1.0078125, "grad_norm": 0.07309086620807648, "learning_rate": 5.833528413179249e-05, "loss": 0.001039872644469142, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00104, "step": 258, "tokens/total": 8372840, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 117363} +{"epoch": 1.01171875, "grad_norm": 0.27652719616889954, "learning_rate": 5.80457242789548e-05, "loss": 0.005169637501239777, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00518, "step": 259, "tokens/total": 8409728, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 117860} +{"epoch": 1.015625, "grad_norm": 0.336326539516449, "learning_rate": 5.77560376810977e-05, "loss": 0.005537042859941721, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.00555, "step": 260, "tokens/total": 8442296, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 118324} +{"epoch": 1.01953125, "grad_norm": 0.024021904915571213, "learning_rate": 5.7466236393263005e-05, "loss": 0.000442724849563092, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00044, "step": 261, "tokens/total": 8474568, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 118770} +{"epoch": 1.0234375, "grad_norm": 0.12144053727388382, "learning_rate": 5.717633247526522e-05, "loss": 0.0017637501005083323, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00177, "step": 262, "tokens/total": 8505520, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 119206} +{"epoch": 1.02734375, "grad_norm": 0.18345829844474792, "learning_rate": 5.688633799118971e-05, "loss": 0.0010704364394769073, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00107, "step": 263, "tokens/total": 8535064, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 119610} +{"epoch": 1.03125, "grad_norm": 1.1682981252670288, "learning_rate": 5.659626500889066e-05, "loss": 0.001668532844632864, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00167, "step": 264, "tokens/total": 8563600, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 120080} +{"epoch": 1.03515625, "grad_norm": 0.22360418736934662, "learning_rate": 5.6306125599488905e-05, "loss": 0.003614983521401882, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00362, "step": 265, "tokens/total": 8592648, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 120519} +{"epoch": 1.0390625, "grad_norm": 0.03066769242286682, "learning_rate": 5.601593183686955e-05, "loss": 0.00027472779038362205, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00027, "step": 266, "tokens/total": 8627328, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 120995} +{"epoch": 1.04296875, "grad_norm": 0.49719637632369995, "learning_rate": 5.572569579717961e-05, "loss": 0.005009994842112064, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00502, "step": 267, "tokens/total": 8659760, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 121459} +{"epoch": 1.046875, "grad_norm": 0.06281400471925735, "learning_rate": 5.543542955832538e-05, "loss": 0.0008970646886155009, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.0009, "step": 268, "tokens/total": 8696968, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 121902} +{"epoch": 1.05078125, "grad_norm": 0.1682845950126648, "learning_rate": 5.514514519946986e-05, "loss": 0.0040419334545731544, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00405, "step": 269, "tokens/total": 8735392, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 122399} +{"epoch": 1.0546875, "grad_norm": 0.03463364765048027, "learning_rate": 5.485485480053015e-05, "loss": 0.0003171416756231338, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00032, "step": 270, "tokens/total": 8769840, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 122804} +{"epoch": 1.05859375, "grad_norm": 0.07116048783063889, "learning_rate": 5.4564570441674645e-05, "loss": 0.0006469864747487009, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00065, "step": 271, "tokens/total": 8800648, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 123244} +{"epoch": 1.0625, "grad_norm": 0.2739856243133545, "learning_rate": 5.42743042028204e-05, "loss": 0.0037272425834089518, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00373, "step": 272, "tokens/total": 8832480, "tokens/train_per_sec_per_gpu": 6.8, "tokens/trainable": 123683} +{"epoch": 1.06640625, "grad_norm": 0.07910279929637909, "learning_rate": 5.3984068163130464e-05, "loss": 0.0011748976539820433, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00118, "step": 273, "tokens/total": 8864840, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 124153} +{"epoch": 1.0703125, "grad_norm": 0.09267326444387436, "learning_rate": 5.369387440051111e-05, "loss": 0.0012363013811409473, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00124, "step": 274, "tokens/total": 8897272, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 124616} +{"epoch": 1.07421875, "grad_norm": 0.1881965547800064, "learning_rate": 5.340373499110935e-05, "loss": 0.0014667342184111476, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00147, "step": 275, "tokens/total": 8930688, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 125109} +{"epoch": 1.078125, "grad_norm": 0.03174170106649399, "learning_rate": 5.3113662008810304e-05, "loss": 0.00025901044136844575, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00026, "step": 276, "tokens/total": 8967216, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 125591} +{"epoch": 1.08203125, "grad_norm": 0.13788066804409027, "learning_rate": 5.282366752473479e-05, "loss": 0.0009195133461616933, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00092, "step": 277, "tokens/total": 9001368, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 126065} +{"epoch": 1.0859375, "grad_norm": 0.021275170147418976, "learning_rate": 5.2533763606737005e-05, "loss": 0.00016193788906093687, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00016, "step": 278, "tokens/total": 9036808, "tokens/train_per_sec_per_gpu": 6.91, "tokens/trainable": 126480} +{"epoch": 1.08984375, "grad_norm": 0.13208508491516113, "learning_rate": 5.224396231890232e-05, "loss": 0.000532572390511632, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00053, "step": 279, "tokens/total": 9067840, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 126929} +{"epoch": 1.09375, "grad_norm": 1.1382790803909302, "learning_rate": 5.195427572104522e-05, "loss": 0.0050590462051332, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00507, "step": 280, "tokens/total": 9098936, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 127366} +{"epoch": 1.09765625, "grad_norm": 0.248318612575531, "learning_rate": 5.166471586820751e-05, "loss": 0.0020478060469031334, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00205, "step": 281, "tokens/total": 9133264, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 127831} +{"epoch": 1.1015625, "grad_norm": 0.17265906929969788, "learning_rate": 5.1375294810156615e-05, "loss": 0.0009765516151674092, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00098, "step": 282, "tokens/total": 9165008, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 128278} +{"epoch": 1.10546875, "grad_norm": 0.3399413526058197, "learning_rate": 5.1086024590884144e-05, "loss": 0.0027936026453971863, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.0028, "step": 283, "tokens/total": 9197952, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 128727} +{"epoch": 1.109375, "grad_norm": 0.06349737197160721, "learning_rate": 5.079691724810461e-05, "loss": 0.00047089322470128536, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00047, "step": 284, "tokens/total": 9232576, "tokens/train_per_sec_per_gpu": 7.12, "tokens/trainable": 129162} +{"epoch": 1.11328125, "grad_norm": 0.015247729606926441, "learning_rate": 5.0507984812754684e-05, "loss": 0.00013652059715241194, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00014, "step": 285, "tokens/total": 9264568, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 129659} +{"epoch": 1.1171875, "grad_norm": 0.007670037914067507, "learning_rate": 5.021923930849237e-05, "loss": 7.305377221200615e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00007, "step": 286, "tokens/total": 9296544, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 130145} +{"epoch": 1.12109375, "grad_norm": 0.038899097591638565, "learning_rate": 4.99306927511967e-05, "loss": 0.00024168932577595115, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00024, "step": 287, "tokens/total": 9327936, "tokens/train_per_sec_per_gpu": 10.45, "tokens/trainable": 130585} +{"epoch": 1.125, "grad_norm": 0.16528911888599396, "learning_rate": 4.964235714846775e-05, "loss": 0.0011005771812051535, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.0011, "step": 288, "tokens/total": 9357984, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 131005} +{"epoch": 1.12890625, "grad_norm": 0.011003553867340088, "learning_rate": 4.9354244499126866e-05, "loss": 7.449094846379012e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00007, "step": 289, "tokens/total": 9388664, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 131471} +{"epoch": 1.1328125, "grad_norm": 0.5522642135620117, "learning_rate": 4.90663667927174e-05, "loss": 0.0011794776655733585, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00118, "step": 290, "tokens/total": 9420504, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 131920} +{"epoch": 1.13671875, "grad_norm": 0.015821164473891258, "learning_rate": 4.877873600900581e-05, "loss": 0.00010274435044266284, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.0001, "step": 291, "tokens/total": 9452664, "tokens/train_per_sec_per_gpu": 6.32, "tokens/trainable": 132379} +{"epoch": 1.140625, "grad_norm": 0.0009114170679822564, "learning_rate": 4.849136411748306e-05, "loss": 1.5004813576524612e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00002, "step": 292, "tokens/total": 9488896, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 132851} +{"epoch": 1.14453125, "grad_norm": 0.1425793170928955, "learning_rate": 4.8204263076866574e-05, "loss": 0.0012265999102964997, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00123, "step": 293, "tokens/total": 9522112, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 133351} +{"epoch": 1.1484375, "grad_norm": 0.0020952539052814245, "learning_rate": 4.791744483460251e-05, "loss": 1.3290811693877913e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00001, "step": 294, "tokens/total": 9555632, "tokens/train_per_sec_per_gpu": 6.6, "tokens/trainable": 133784} +{"epoch": 1.15234375, "grad_norm": 0.0006906016496941447, "learning_rate": 4.7630921326368736e-05, "loss": 1.335429806204047e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00001, "step": 295, "tokens/total": 9590904, "tokens/train_per_sec_per_gpu": 6.44, "tokens/trainable": 134209} +{"epoch": 1.15625, "grad_norm": 0.10362102091312408, "learning_rate": 4.7344704475577916e-05, "loss": 0.0006106112850829959, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00061, "step": 296, "tokens/total": 9620288, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 134645} +{"epoch": 1.16015625, "grad_norm": 0.10874857753515244, "learning_rate": 4.705880619288153e-05, "loss": 0.0006240932852961123, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00062, "step": 297, "tokens/total": 9651984, "tokens/train_per_sec_per_gpu": 7.12, "tokens/trainable": 135047} +{"epoch": 1.1640625, "grad_norm": 0.00565246632322669, "learning_rate": 4.677323837567412e-05, "loss": 4.0772894863039255e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00004, "step": 298, "tokens/total": 9681368, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 135474} +{"epoch": 1.16796875, "grad_norm": 0.10413940995931625, "learning_rate": 4.6488012907598146e-05, "loss": 7.811943942215294e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00008, "step": 299, "tokens/total": 9712872, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 135948} +{"epoch": 1.171875, "grad_norm": 0.4719710350036621, "learning_rate": 4.620314165804964e-05, "loss": 0.004332612734287977, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00434, "step": 300, "tokens/total": 9745184, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 136421} +{"epoch": 1.17578125, "grad_norm": 0.008070503361523151, "learning_rate": 4.591863648168407e-05, "loss": 6.053931429050863e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00006, "step": 301, "tokens/total": 9778928, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 136886} +{"epoch": 1.1796875, "grad_norm": 0.002919843653216958, "learning_rate": 4.5634509217923135e-05, "loss": 1.3332632079254836e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00001, "step": 302, "tokens/total": 9810752, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 137321} +{"epoch": 1.18359375, "grad_norm": 0.0016050691483542323, "learning_rate": 4.535077169046201e-05, "loss": 2.752063301159069e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00003, "step": 303, "tokens/total": 9843672, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 137758} +{"epoch": 1.1875, "grad_norm": 0.41201603412628174, "learning_rate": 4.506743570677743e-05, "loss": 0.01910705864429474, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.01929, "step": 304, "tokens/total": 9873752, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 138190} +{"epoch": 1.19140625, "grad_norm": 0.008029904216527939, "learning_rate": 4.478451305763618e-05, "loss": 2.008186129387468e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00002, "step": 305, "tokens/total": 9906272, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 138650} +{"epoch": 1.1953125, "grad_norm": 0.0006128453533165157, "learning_rate": 4.450201551660454e-05, "loss": 1.3922262041887734e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00001, "step": 306, "tokens/total": 9936296, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 139082} +{"epoch": 1.19921875, "grad_norm": 0.8921307325363159, "learning_rate": 4.4219954839558276e-05, "loss": 0.004690371919423342, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0047, "step": 307, "tokens/total": 9967720, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 139523} +{"epoch": 1.203125, "grad_norm": 0.29221948981285095, "learning_rate": 4.393834276419352e-05, "loss": 0.0018672687001526356, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.00187, "step": 308, "tokens/total": 9997176, "tokens/train_per_sec_per_gpu": 10.83, "tokens/trainable": 140010} +{"epoch": 1.20703125, "grad_norm": 1.1316039562225342, "learning_rate": 4.36571910095382e-05, "loss": 0.0328356958925724, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.03338, "step": 309, "tokens/total": 10031872, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 140454} +{"epoch": 1.2109375, "grad_norm": 0.010447807610034943, "learning_rate": 4.337651127546448e-05, "loss": 0.00010874496365431696, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00011, "step": 310, "tokens/total": 10065240, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 140905} +{"epoch": 1.21484375, "grad_norm": 0.011686180718243122, "learning_rate": 4.3096315242201736e-05, "loss": 0.00011199204163858667, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00011, "step": 311, "tokens/total": 10096640, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 141339} +{"epoch": 1.21875, "grad_norm": 0.22617779672145844, "learning_rate": 4.2816614569850635e-05, "loss": 0.0024244196247309446, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00243, "step": 312, "tokens/total": 10130192, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 141778} +{"epoch": 1.22265625, "grad_norm": 0.04222942143678665, "learning_rate": 4.2537420897897864e-05, "loss": 0.00034038140438497066, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00034, "step": 313, "tokens/total": 10164248, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 142255} +{"epoch": 1.2265625, "grad_norm": 0.01760595291852951, "learning_rate": 4.225874584473174e-05, "loss": 0.00018935283878818154, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00019, "step": 314, "tokens/total": 10194664, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 142693} +{"epoch": 1.23046875, "grad_norm": 0.007313643116503954, "learning_rate": 4.19806010071587e-05, "loss": 0.00016455976583529264, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00016, "step": 315, "tokens/total": 10227768, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 143157} +{"epoch": 1.234375, "grad_norm": 0.23040907084941864, "learning_rate": 4.170299795992081e-05, "loss": 0.002754031214863062, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00276, "step": 316, "tokens/total": 10260464, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 143603} +{"epoch": 1.23828125, "grad_norm": 0.04454594850540161, "learning_rate": 4.142594825521398e-05, "loss": 0.0006025090115144849, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.0006, "step": 317, "tokens/total": 10297528, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 144084} +{"epoch": 1.2421875, "grad_norm": 0.1110771894454956, "learning_rate": 4.114946342220728e-05, "loss": 0.0004911564174108207, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00049, "step": 318, "tokens/total": 10331448, "tokens/train_per_sec_per_gpu": 7.0, "tokens/trainable": 144532} +{"epoch": 1.24609375, "grad_norm": 0.016842398792505264, "learning_rate": 4.087355496656321e-05, "loss": 0.00038182971184141934, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00038, "step": 319, "tokens/total": 10363152, "tokens/train_per_sec_per_gpu": 10.2, "tokens/trainable": 145004} +{"epoch": 1.25, "grad_norm": 0.668603777885437, "learning_rate": 4.05982343699588e-05, "loss": 0.004701887723058462, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00471, "step": 320, "tokens/total": 10397832, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 145498} +{"epoch": 1.25390625, "grad_norm": 0.7681360244750977, "learning_rate": 4.0323513089607876e-05, "loss": 0.0018627303652465343, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00186, "step": 321, "tokens/total": 10431416, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 145965} +{"epoch": 1.2578125, "grad_norm": 0.022167202085256577, "learning_rate": 4.004940255778431e-05, "loss": 0.0003874839167110622, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00039, "step": 322, "tokens/total": 10467384, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 146409} +{"epoch": 1.26171875, "grad_norm": 0.14123600721359253, "learning_rate": 3.977591418134619e-05, "loss": 0.0010417590383440256, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00104, "step": 323, "tokens/total": 10500792, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 146849} +{"epoch": 1.265625, "grad_norm": 0.06649359315633774, "learning_rate": 3.95030593412612e-05, "loss": 0.0007338112336583436, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00073, "step": 324, "tokens/total": 10532592, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 147272} +{"epoch": 1.26953125, "grad_norm": 0.05515127256512642, "learning_rate": 3.923084939213296e-05, "loss": 0.0005398925859481096, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00054, "step": 325, "tokens/total": 10562392, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 147694} +{"epoch": 1.2734375, "grad_norm": 0.5470691323280334, "learning_rate": 3.895929566172861e-05, "loss": 0.006047589238733053, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00607, "step": 326, "tokens/total": 10595416, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 148158} +{"epoch": 1.27734375, "grad_norm": 0.022788170725107193, "learning_rate": 3.868840945050728e-05, "loss": 0.00029666523914784193, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.0003, "step": 327, "tokens/total": 10630504, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 148590} +{"epoch": 1.28125, "grad_norm": 0.6924564242362976, "learning_rate": 3.841820203114995e-05, "loss": 0.002671848051249981, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00268, "step": 328, "tokens/total": 10661632, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 149037} +{"epoch": 1.28515625, "grad_norm": 0.11377784609794617, "learning_rate": 3.814868464809027e-05, "loss": 0.0005837328499183059, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00058, "step": 329, "tokens/total": 10691544, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 149439} +{"epoch": 1.2890625, "grad_norm": 0.3690112233161926, "learning_rate": 3.787986851704667e-05, "loss": 0.004535990301519632, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00455, "step": 330, "tokens/total": 10724768, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 149869} +{"epoch": 1.29296875, "grad_norm": 0.04244185984134674, "learning_rate": 3.7611764824555654e-05, "loss": 0.00032598705729469657, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00033, "step": 331, "tokens/total": 10756008, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 150343} +{"epoch": 1.296875, "grad_norm": 0.016152439638972282, "learning_rate": 3.734438472750619e-05, "loss": 0.00021058136189822108, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00021, "step": 332, "tokens/total": 10793152, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 150814} +{"epoch": 1.30078125, "grad_norm": 0.02136923186480999, "learning_rate": 3.707773935267552e-05, "loss": 0.00020714342826977372, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00021, "step": 333, "tokens/total": 10827624, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 151300} +{"epoch": 1.3046875, "grad_norm": 0.008657954633235931, "learning_rate": 3.68118397962661e-05, "loss": 0.00016192265320569277, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00016, "step": 334, "tokens/total": 10859712, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 151708} +{"epoch": 1.30859375, "grad_norm": 0.009751328267157078, "learning_rate": 3.654669712344384e-05, "loss": 0.00011420920782256871, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00011, "step": 335, "tokens/total": 10888792, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 152164} +{"epoch": 1.3125, "grad_norm": 0.04729770869016647, "learning_rate": 3.628232236787763e-05, "loss": 0.0002625104389153421, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00026, "step": 336, "tokens/total": 10922968, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 152603} +{"epoch": 1.31640625, "grad_norm": 0.010269912891089916, "learning_rate": 3.6018726531280144e-05, "loss": 0.00016116736514959484, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00016, "step": 337, "tokens/total": 10958000, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 153094} +{"epoch": 1.3203125, "grad_norm": 0.03481622785329819, "learning_rate": 3.575592058295017e-05, "loss": 0.00017682835459709167, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00018, "step": 338, "tokens/total": 10989872, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 153573} +{"epoch": 1.32421875, "grad_norm": 0.0025050619151443243, "learning_rate": 3.549391545931585e-05, "loss": 5.5833028454799205e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00006, "step": 339, "tokens/total": 11020784, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 154033} +{"epoch": 1.328125, "grad_norm": 0.02628808096051216, "learning_rate": 3.5232722063479914e-05, "loss": 0.00014375323371496052, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00014, "step": 340, "tokens/total": 11053240, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 154482} +{"epoch": 1.33203125, "grad_norm": 1.194659948348999, "learning_rate": 3.49723512647657e-05, "loss": 0.0034497627057135105, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00346, "step": 341, "tokens/total": 11089040, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 154982} +{"epoch": 1.3359375, "grad_norm": 0.22251085937023163, "learning_rate": 3.471281389826491e-05, "loss": 0.0008216641726903617, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00082, "step": 342, "tokens/total": 11119672, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 155387} +{"epoch": 1.33984375, "grad_norm": 0.021134935319423676, "learning_rate": 3.4454120764386764e-05, "loss": 0.00020119277178309858, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.0002, "step": 343, "tokens/total": 11155040, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 155842} +{"epoch": 1.34375, "grad_norm": 0.014288877137005329, "learning_rate": 3.4196282628408526e-05, "loss": 0.00018723135872278363, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00019, "step": 344, "tokens/total": 11185952, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 156287} +{"epoch": 1.34765625, "grad_norm": 0.006567079573869705, "learning_rate": 3.3939310220027456e-05, "loss": 0.00012955436250194907, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00013, "step": 345, "tokens/total": 11219272, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 156782} +{"epoch": 1.3515625, "grad_norm": 0.016391292214393616, "learning_rate": 3.3683214232914404e-05, "loss": 9.264260006602854e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00009, "step": 346, "tokens/total": 11253424, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 157255} +{"epoch": 1.35546875, "grad_norm": 0.27196013927459717, "learning_rate": 3.342800532426873e-05, "loss": 0.002653596457093954, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00266, "step": 347, "tokens/total": 11286216, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 157721} +{"epoch": 1.359375, "grad_norm": 0.2805938720703125, "learning_rate": 3.317369411437484e-05, "loss": 0.0047271111980080605, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00474, "step": 348, "tokens/total": 11319480, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 158189} +{"epoch": 1.36328125, "grad_norm": 0.16783228516578674, "learning_rate": 3.292029118616024e-05, "loss": 0.0006079705781303346, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00061, "step": 349, "tokens/total": 11350472, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 158612} +{"epoch": 1.3671875, "grad_norm": 0.06430937349796295, "learning_rate": 3.266780708475511e-05, "loss": 0.0005609479267150164, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00056, "step": 350, "tokens/total": 11383088, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 159090} +{"epoch": 1.37109375, "grad_norm": 0.08470334857702255, "learning_rate": 3.241625231705354e-05, "loss": 0.00034375250106677413, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00034, "step": 351, "tokens/total": 11418392, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 159549} +{"epoch": 1.375, "grad_norm": 0.0049989852122962475, "learning_rate": 3.216563735127618e-05, "loss": 5.816295379190706e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00006, "step": 352, "tokens/total": 11449784, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 160026} +{"epoch": 1.37890625, "grad_norm": 0.01120883971452713, "learning_rate": 3.191597261653475e-05, "loss": 0.0001530043373350054, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00015, "step": 353, "tokens/total": 11484240, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 160487} +{"epoch": 1.3828125, "grad_norm": 0.492365300655365, "learning_rate": 3.166726850239794e-05, "loss": 0.0025269677862524986, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00253, "step": 354, "tokens/total": 11513360, "tokens/train_per_sec_per_gpu": 11.03, "tokens/trainable": 160935} +{"epoch": 1.38671875, "grad_norm": 0.260793536901474, "learning_rate": 3.141953535845912e-05, "loss": 0.001510852249339223, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00151, "step": 355, "tokens/total": 11543112, "tokens/train_per_sec_per_gpu": 10.31, "tokens/trainable": 161385} +{"epoch": 1.390625, "grad_norm": 0.022635826840996742, "learning_rate": 3.11727834939056e-05, "loss": 0.0002208410733146593, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00022, "step": 356, "tokens/total": 11573464, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 161849} +{"epoch": 1.39453125, "grad_norm": 0.4709544777870178, "learning_rate": 3.092702317708967e-05, "loss": 0.00713586388155818, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00716, "step": 357, "tokens/total": 11604896, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 162306} +{"epoch": 1.3984375, "grad_norm": 0.020246008411049843, "learning_rate": 3.0682264635101276e-05, "loss": 0.00018354528583586216, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00018, "step": 358, "tokens/total": 11636368, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 162759} +{"epoch": 1.40234375, "grad_norm": 0.00750375771895051, "learning_rate": 3.0438518053342407e-05, "loss": 7.945183460833505e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00008, "step": 359, "tokens/total": 11670216, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 163230} +{"epoch": 1.40625, "grad_norm": 0.015559608116745949, "learning_rate": 3.0195793575103266e-05, "loss": 0.00014182804443407804, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00014, "step": 360, "tokens/total": 11707816, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 163694} +{"epoch": 1.41015625, "grad_norm": 0.21449947357177734, "learning_rate": 2.9954101301140146e-05, "loss": 0.000996988732367754, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.001, "step": 361, "tokens/total": 11742584, "tokens/train_per_sec_per_gpu": 6.93, "tokens/trainable": 164144} +{"epoch": 1.4140625, "grad_norm": 0.04091532155871391, "learning_rate": 2.9713451289255123e-05, "loss": 0.0003563946229405701, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00036, "step": 362, "tokens/total": 11770384, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 164585} +{"epoch": 1.41796875, "grad_norm": 0.033561985939741135, "learning_rate": 2.9473853553877484e-05, "loss": 0.00017687503714114428, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00018, "step": 363, "tokens/total": 11803208, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 165059} +{"epoch": 1.421875, "grad_norm": 0.00818592868745327, "learning_rate": 2.9235318065647e-05, "loss": 0.0001041634677676484, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.0001, "step": 364, "tokens/total": 11840480, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 165519} +{"epoch": 1.42578125, "grad_norm": 0.011228879913687706, "learning_rate": 2.8997854750998964e-05, "loss": 9.578435856383294e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.0001, "step": 365, "tokens/total": 11875544, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 165984} +{"epoch": 1.4296875, "grad_norm": 0.29387128353118896, "learning_rate": 2.8761473491751258e-05, "loss": 0.0014026440912857652, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.0014, "step": 366, "tokens/total": 11907656, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 166419} +{"epoch": 1.43359375, "grad_norm": 0.1449151188135147, "learning_rate": 2.8526184124692883e-05, "loss": 0.0009720182861201465, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00097, "step": 367, "tokens/total": 11941904, "tokens/train_per_sec_per_gpu": 6.42, "tokens/trainable": 166857} +{"epoch": 1.4375, "grad_norm": 0.0069118100218474865, "learning_rate": 2.829199644117484e-05, "loss": 9.797836537472904e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.0001, "step": 368, "tokens/total": 11973624, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 167267} +{"epoch": 1.44140625, "grad_norm": 0.03757087513804436, "learning_rate": 2.8058920186702553e-05, "loss": 0.00013070915883872658, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00013, "step": 369, "tokens/total": 12010360, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 167740} +{"epoch": 1.4453125, "grad_norm": 0.007039215881377459, "learning_rate": 2.782696506053033e-05, "loss": 7.435551378875971e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00007, "step": 370, "tokens/total": 12044544, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 168172} +{"epoch": 1.44921875, "grad_norm": 0.04343387112021446, "learning_rate": 2.7596140715257824e-05, "loss": 0.0002924727741628885, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00029, "step": 371, "tokens/total": 12075792, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 168637} +{"epoch": 1.453125, "grad_norm": 0.016366302967071533, "learning_rate": 2.7366456756428184e-05, "loss": 5.919451723457314e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00006, "step": 372, "tokens/total": 12108088, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 169094} +{"epoch": 1.45703125, "grad_norm": 0.0025336267426609993, "learning_rate": 2.7137922742128486e-05, "loss": 2.3547287128167227e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00002, "step": 373, "tokens/total": 12140888, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 169593} +{"epoch": 1.4609375, "grad_norm": 0.008316479623317719, "learning_rate": 2.691054818259188e-05, "loss": 5.584599057328887e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00006, "step": 374, "tokens/total": 12174560, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 170066} +{"epoch": 1.46484375, "grad_norm": 0.1824461668729782, "learning_rate": 2.6684342539801933e-05, "loss": 0.0005074248765595257, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00051, "step": 375, "tokens/total": 12204848, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 170518} +{"epoch": 1.46875, "grad_norm": 0.28998851776123047, "learning_rate": 2.645931522709877e-05, "loss": 0.0013615130446851254, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00136, "step": 376, "tokens/total": 12238472, "tokens/train_per_sec_per_gpu": 10.8, "tokens/trainable": 170998} +{"epoch": 1.47265625, "grad_norm": 0.6462250351905823, "learning_rate": 2.6235475608787365e-05, "loss": 0.004619921557605267, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00463, "step": 377, "tokens/total": 12271024, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 171459} +{"epoch": 1.4765625, "grad_norm": 0.03151080012321472, "learning_rate": 2.6012832999747916e-05, "loss": 0.00020574626978486776, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00021, "step": 378, "tokens/total": 12301312, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 171905} +{"epoch": 1.48046875, "grad_norm": 0.8245978951454163, "learning_rate": 2.579139666504821e-05, "loss": 0.02442801184952259, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.02473, "step": 379, "tokens/total": 12337072, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 172351} +{"epoch": 1.484375, "grad_norm": 0.010328572243452072, "learning_rate": 2.557117581955798e-05, "loss": 5.412324026110582e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00005, "step": 380, "tokens/total": 12368000, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 172817} +{"epoch": 1.48828125, "grad_norm": 0.5278563499450684, "learning_rate": 2.5352179627565532e-05, "loss": 0.009411942213773727, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00946, "step": 381, "tokens/total": 12399880, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 173258} +{"epoch": 1.4921875, "grad_norm": 0.022976579144597054, "learning_rate": 2.5134417202396277e-05, "loss": 3.591702625271864e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00004, "step": 382, "tokens/total": 12432288, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 173665} +{"epoch": 1.49609375, "grad_norm": 0.011402076110243797, "learning_rate": 2.491789760603361e-05, "loss": 9.239866631105542e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00009, "step": 383, "tokens/total": 12465592, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 174095} +{"epoch": 1.5, "grad_norm": 0.3607484996318817, "learning_rate": 2.4702629848741764e-05, "loss": 0.0017198571003973484, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00172, "step": 384, "tokens/total": 12498200, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 174550} +{"epoch": 1.50390625, "grad_norm": 0.08872281759977341, "learning_rate": 2.4488622888690785e-05, "loss": 0.00014599430141970515, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00015, "step": 385, "tokens/total": 12530280, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 174990} +{"epoch": 1.5078125, "grad_norm": 0.006218935828655958, "learning_rate": 2.427588563158384e-05, "loss": 4.5289274567039683e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00005, "step": 386, "tokens/total": 12563360, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 175476} +{"epoch": 1.51171875, "grad_norm": 0.017305009067058563, "learning_rate": 2.406442693028651e-05, "loss": 0.00013526745897252113, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00014, "step": 387, "tokens/total": 12593032, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 175897} +{"epoch": 1.515625, "grad_norm": 0.005622038617730141, "learning_rate": 2.3854255584458547e-05, "loss": 6.211179425008595e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00006, "step": 388, "tokens/total": 12628272, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 176324} +{"epoch": 1.51953125, "grad_norm": 0.0313546247780323, "learning_rate": 2.3645380340187508e-05, "loss": 0.00010915705206571147, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00011, "step": 389, "tokens/total": 12661368, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 176805} +{"epoch": 1.5234375, "grad_norm": 0.007129088509827852, "learning_rate": 2.3437809889624914e-05, "loss": 0.00010077482875203714, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.77, "memory/max_allocated (GiB)": 63.77, "ppl": 1.0001, "step": 390, "tokens/total": 12691168, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 177256} +{"epoch": 1.52734375, "grad_norm": 0.006361784413456917, "learning_rate": 2.3231552870624487e-05, "loss": 6.839673733338714e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00007, "step": 391, "tokens/total": 12721264, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 177682} +{"epoch": 1.53125, "grad_norm": 0.004127918742597103, "learning_rate": 2.3026617866382657e-05, "loss": 5.998688720865175e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00006, "step": 392, "tokens/total": 12753640, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 178179} +{"epoch": 1.53515625, "grad_norm": 0.11638103425502777, "learning_rate": 2.2823013405081507e-05, "loss": 0.0002966455649584532, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.0003, "step": 393, "tokens/total": 12787056, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 178676} +{"epoch": 1.5390625, "grad_norm": 0.011365041136741638, "learning_rate": 2.2620747959533722e-05, "loss": 8.411706221522763e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00008, "step": 394, "tokens/total": 12819312, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 179173} +{"epoch": 1.54296875, "grad_norm": 0.03144831955432892, "learning_rate": 2.2419829946830123e-05, "loss": 0.00010599827510304749, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00011, "step": 395, "tokens/total": 12853376, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 179624} +{"epoch": 1.546875, "grad_norm": 0.010500593110918999, "learning_rate": 2.2220267727989325e-05, "loss": 7.906509563326836e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00008, "step": 396, "tokens/total": 12882960, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 180078} +{"epoch": 1.55078125, "grad_norm": 0.00721990130841732, "learning_rate": 2.202206960760984e-05, "loss": 0.00010571345046628267, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00011, "step": 397, "tokens/total": 12917016, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 180585} +{"epoch": 1.5546875, "grad_norm": 0.00871045608073473, "learning_rate": 2.182524383352446e-05, "loss": 7.464267400791869e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00007, "step": 398, "tokens/total": 12950056, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 181039} +{"epoch": 1.55859375, "grad_norm": 0.6837775707244873, "learning_rate": 2.1629798596457056e-05, "loss": 0.011667979881167412, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.01174, "step": 399, "tokens/total": 12981728, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 181489} +{"epoch": 1.5625, "grad_norm": 0.004655770026147366, "learning_rate": 2.1435742029681725e-05, "loss": 6.092998228268698e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00006, "step": 400, "tokens/total": 13011464, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 181906} +{"epoch": 1.56640625, "grad_norm": 0.0050313775427639484, "learning_rate": 2.124308220868431e-05, "loss": 4.7873571020318195e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00005, "step": 401, "tokens/total": 13044432, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 182361} +{"epoch": 1.5703125, "grad_norm": 0.1338331550359726, "learning_rate": 2.105182715082638e-05, "loss": 0.0007508323178626597, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00075, "step": 402, "tokens/total": 13075608, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 182838} +{"epoch": 1.57421875, "grad_norm": 0.04298929497599602, "learning_rate": 2.0861984815011552e-05, "loss": 0.00042654373100958765, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00043, "step": 403, "tokens/total": 13107352, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 183269} +{"epoch": 1.578125, "grad_norm": 0.009988275356590748, "learning_rate": 2.0673563101354323e-05, "loss": 9.791548654902726e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.0001, "step": 404, "tokens/total": 13140736, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 183699} +{"epoch": 1.58203125, "grad_norm": 0.062436673790216446, "learning_rate": 2.0486569850851317e-05, "loss": 0.000217036169487983, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00022, "step": 405, "tokens/total": 13174040, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 184162} +{"epoch": 1.5859375, "grad_norm": 0.1971691995859146, "learning_rate": 2.0301012845054956e-05, "loss": 0.0009794795187190175, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00098, "step": 406, "tokens/total": 13207232, "tokens/train_per_sec_per_gpu": 6.61, "tokens/trainable": 184576} +{"epoch": 1.58984375, "grad_norm": 0.006136247888207436, "learning_rate": 2.011689980574966e-05, "loss": 9.234969911631197e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00009, "step": 407, "tokens/total": 13241160, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 184995} +{"epoch": 1.59375, "grad_norm": 0.2965022623538971, "learning_rate": 1.993423839463052e-05, "loss": 0.0009314694325439632, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00093, "step": 408, "tokens/total": 13273104, "tokens/train_per_sec_per_gpu": 10.46, "tokens/trainable": 185452} +{"epoch": 1.59765625, "grad_norm": 0.019490495324134827, "learning_rate": 1.975303621298445e-05, "loss": 0.00014213283429853618, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00014, "step": 409, "tokens/total": 13308136, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 185904} +{"epoch": 1.6015625, "grad_norm": 0.13800953328609467, "learning_rate": 1.957330080137385e-05, "loss": 0.0008366279653273523, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00084, "step": 410, "tokens/total": 13337816, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 186355} +{"epoch": 1.60546875, "grad_norm": 0.06092296168208122, "learning_rate": 1.9395039639322864e-05, "loss": 0.00028753624064847827, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00029, "step": 411, "tokens/total": 13372576, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 186831} +{"epoch": 1.609375, "grad_norm": 0.0011055499780923128, "learning_rate": 1.9218260145006073e-05, "loss": 3.157389073749073e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00003, "step": 412, "tokens/total": 13406664, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 187317} +{"epoch": 1.61328125, "grad_norm": 0.008461332879960537, "learning_rate": 1.904296967493982e-05, "loss": 8.71939628268592e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00009, "step": 413, "tokens/total": 13438256, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 187773} +{"epoch": 1.6171875, "grad_norm": 0.3203028440475464, "learning_rate": 1.8869175523676064e-05, "loss": 0.004225490614771843, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00423, "step": 414, "tokens/total": 13472096, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 188268} +{"epoch": 1.62109375, "grad_norm": 0.0037612791638821363, "learning_rate": 1.869688492349885e-05, "loss": 8.429785521002486e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00008, "step": 415, "tokens/total": 13504192, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 188720} +{"epoch": 1.625, "grad_norm": 0.0380038321018219, "learning_rate": 1.85261050441233e-05, "loss": 0.0002530024212319404, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00025, "step": 416, "tokens/total": 13537392, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 189177} +{"epoch": 1.62890625, "grad_norm": 1.1339021921157837, "learning_rate": 1.8356842992397304e-05, "loss": 0.007872459478676319, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.0079, "step": 417, "tokens/total": 13569344, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 189617} +{"epoch": 1.6328125, "grad_norm": 0.5922052264213562, "learning_rate": 1.8189105812005714e-05, "loss": 0.0016181766986846924, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00162, "step": 418, "tokens/total": 13602200, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 190028} +{"epoch": 1.63671875, "grad_norm": 0.5275221467018127, "learning_rate": 1.802290048317732e-05, "loss": 0.002003616187721491, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00201, "step": 419, "tokens/total": 13632640, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 190445} +{"epoch": 1.640625, "grad_norm": 0.006155412178486586, "learning_rate": 1.785823392239424e-05, "loss": 6.299345113802701e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00006, "step": 420, "tokens/total": 13664920, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 190886} +{"epoch": 1.64453125, "grad_norm": 0.004714473150670528, "learning_rate": 1.7695112982104225e-05, "loss": 6.728777952957898e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00007, "step": 421, "tokens/total": 13697144, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 191325} +{"epoch": 1.6484375, "grad_norm": 0.004165345337241888, "learning_rate": 1.7533544450435433e-05, "loss": 4.8686844820622355e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00005, "step": 422, "tokens/total": 13724496, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 191758} +{"epoch": 1.65234375, "grad_norm": 0.003542484249919653, "learning_rate": 1.7373535050913946e-05, "loss": 5.6113669415935874e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00006, "step": 423, "tokens/total": 13757880, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 192195} +{"epoch": 1.65625, "grad_norm": 0.010269145481288433, "learning_rate": 1.721509144218405e-05, "loss": 5.334365414455533e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00005, "step": 424, "tokens/total": 13787648, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 192625} +{"epoch": 1.66015625, "grad_norm": 0.0017698201118037105, "learning_rate": 1.705822021773101e-05, "loss": 3.1410152587341145e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00003, "step": 425, "tokens/total": 13819720, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 193111} +{"epoch": 1.6640625, "grad_norm": 1.452236294746399, "learning_rate": 1.69029279056068e-05, "loss": 0.01433581579476595, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.01444, "step": 426, "tokens/total": 13854944, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 193551} +{"epoch": 1.66796875, "grad_norm": 0.389563649892807, "learning_rate": 1.6749220968158415e-05, "loss": 0.0017003817483782768, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.0017, "step": 427, "tokens/total": 13887824, "tokens/train_per_sec_per_gpu": 10.39, "tokens/trainable": 194057} +{"epoch": 1.671875, "grad_norm": 0.1405966430902481, "learning_rate": 1.659710580175893e-05, "loss": 0.000629978661891073, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00063, "step": 428, "tokens/total": 13918192, "tokens/train_per_sec_per_gpu": 10.17, "tokens/trainable": 194556} +{"epoch": 1.67578125, "grad_norm": 0.006760572083294392, "learning_rate": 1.644658873654133e-05, "loss": 0.00010012283746618778, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.0001, "step": 429, "tokens/total": 13947440, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 194993} +{"epoch": 1.6796875, "grad_norm": 0.007503984961658716, "learning_rate": 1.629767603613508e-05, "loss": 8.316665480379015e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00008, "step": 430, "tokens/total": 13974360, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 195393} +{"epoch": 1.68359375, "grad_norm": 0.053249116986989975, "learning_rate": 1.615037389740547e-05, "loss": 0.0002977595722768456, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.0003, "step": 431, "tokens/total": 14006592, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 195837} +{"epoch": 1.6875, "grad_norm": 0.011923098936676979, "learning_rate": 1.600468845019576e-05, "loss": 0.00015089561929926276, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00015, "step": 432, "tokens/total": 14040776, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 196317} +{"epoch": 1.69140625, "grad_norm": 0.0076702069491147995, "learning_rate": 1.5860625757072092e-05, "loss": 6.015443796059117e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00006, "step": 433, "tokens/total": 14073040, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 196765} +{"epoch": 1.6953125, "grad_norm": 0.3260628283023834, "learning_rate": 1.571819181307116e-05, "loss": 0.0008977774996310472, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.0009, "step": 434, "tokens/total": 14107624, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 197237} +{"epoch": 1.69921875, "grad_norm": 0.28617340326309204, "learning_rate": 1.557739254545075e-05, "loss": 0.0013769015204161406, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00138, "step": 435, "tokens/total": 14137784, "tokens/train_per_sec_per_gpu": 10.67, "tokens/trainable": 197709} +{"epoch": 1.703125, "grad_norm": 0.033369503915309906, "learning_rate": 1.543823381344311e-05, "loss": 0.0002449168241582811, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00024, "step": 436, "tokens/total": 14167776, "tokens/train_per_sec_per_gpu": 10.8, "tokens/trainable": 198184} +{"epoch": 1.70703125, "grad_norm": 0.05052034929394722, "learning_rate": 1.5300721408011114e-05, "loss": 0.0002975638199131936, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.0003, "step": 437, "tokens/total": 14199752, "tokens/train_per_sec_per_gpu": 10.24, "tokens/trainable": 198641} +{"epoch": 1.7109375, "grad_norm": 0.8356660604476929, "learning_rate": 1.5164861051607254e-05, "loss": 0.007404692936688662, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.00743, "step": 438, "tokens/total": 14230344, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 199049} +{"epoch": 1.71484375, "grad_norm": 0.0032267768401652575, "learning_rate": 1.5030658397935521e-05, "loss": 4.06214821850881e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00004, "step": 439, "tokens/total": 14263808, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 199518} +{"epoch": 1.71875, "grad_norm": 0.019810471683740616, "learning_rate": 1.4898119031716104e-05, "loss": 0.00025049212854355574, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00025, "step": 440, "tokens/total": 14295864, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 199976} +{"epoch": 1.72265625, "grad_norm": 0.006710933521389961, "learning_rate": 1.476724846845306e-05, "loss": 5.772791337221861e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00006, "step": 441, "tokens/total": 14331688, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 200453} +{"epoch": 1.7265625, "grad_norm": 0.09225372225046158, "learning_rate": 1.463805215420471e-05, "loss": 0.0002776853507384658, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00028, "step": 442, "tokens/total": 14364688, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 200923} +{"epoch": 1.73046875, "grad_norm": 1.209424614906311, "learning_rate": 1.451053546535705e-05, "loss": 0.006454454734921455, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00648, "step": 443, "tokens/total": 14397832, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 201379} +{"epoch": 1.734375, "grad_norm": 0.021958287805318832, "learning_rate": 1.438470370840001e-05, "loss": 0.00014199646830093116, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00014, "step": 444, "tokens/total": 14431864, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 201851} +{"epoch": 1.73828125, "grad_norm": 0.6219879984855652, "learning_rate": 1.4260562119706606e-05, "loss": 0.002361792605370283, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00236, "step": 445, "tokens/total": 14462152, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 202325} +{"epoch": 1.7421875, "grad_norm": 0.004639413207769394, "learning_rate": 1.413811586531508e-05, "loss": 7.780381565680727e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00008, "step": 446, "tokens/total": 14493072, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 202766} +{"epoch": 1.74609375, "grad_norm": 0.35009220242500305, "learning_rate": 1.4017370040713884e-05, "loss": 0.0011031532194465399, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.0011, "step": 447, "tokens/total": 14527392, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 203215} +{"epoch": 1.75, "grad_norm": 0.06996803730726242, "learning_rate": 1.3898329670629645e-05, "loss": 0.0004028659895993769, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.0004, "step": 448, "tokens/total": 14561992, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 203660} +{"epoch": 1.75390625, "grad_norm": 0.004205971956253052, "learning_rate": 1.3780999708818058e-05, "loss": 6.61477170069702e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00007, "step": 449, "tokens/total": 14592056, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 204110} +{"epoch": 1.7578125, "grad_norm": 0.015404323115944862, "learning_rate": 1.3665385037857758e-05, "loss": 0.00017800375644583255, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00018, "step": 450, "tokens/total": 14622816, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 204562} +{"epoch": 1.76171875, "grad_norm": 0.11312013119459152, "learning_rate": 1.3551490468947126e-05, "loss": 0.0011081889970228076, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00111, "step": 451, "tokens/total": 14653272, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 205008} +{"epoch": 1.765625, "grad_norm": 0.6815623641014099, "learning_rate": 1.3439320741704075e-05, "loss": 0.006332481279969215, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00635, "step": 452, "tokens/total": 14683952, "tokens/train_per_sec_per_gpu": 10.07, "tokens/trainable": 205432} +{"epoch": 1.76953125, "grad_norm": 0.0125403618440032, "learning_rate": 1.3328880523968808e-05, "loss": 0.00012937310384586453, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00013, "step": 453, "tokens/total": 14716256, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 205895} +{"epoch": 1.7734375, "grad_norm": 0.05745629221200943, "learning_rate": 1.3220174411609587e-05, "loss": 0.0003890590451192111, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00039, "step": 454, "tokens/total": 14749088, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 206334} +{"epoch": 1.77734375, "grad_norm": 0.007371814455837011, "learning_rate": 1.3113206928331471e-05, "loss": 0.00011688619269989431, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00012, "step": 455, "tokens/total": 14777384, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 206796} +{"epoch": 1.78125, "grad_norm": 0.11222444474697113, "learning_rate": 1.300798252548806e-05, "loss": 0.00018539624579716474, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00019, "step": 456, "tokens/total": 14809640, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 207248} +{"epoch": 1.78515625, "grad_norm": 0.2724775969982147, "learning_rate": 1.2904505581896265e-05, "loss": 0.010108325630426407, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01016, "step": 457, "tokens/total": 14843728, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 207721} +{"epoch": 1.7890625, "grad_norm": 0.016691960394382477, "learning_rate": 1.2802780403654082e-05, "loss": 0.00015424133744090796, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00015, "step": 458, "tokens/total": 14876256, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 208168} +{"epoch": 1.79296875, "grad_norm": 0.012959704734385014, "learning_rate": 1.2702811223961408e-05, "loss": 0.00017573863442521542, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00018, "step": 459, "tokens/total": 14907000, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 208629} +{"epoch": 1.796875, "grad_norm": 0.008339573629200459, "learning_rate": 1.2604602202943861e-05, "loss": 0.00011004183033946902, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00011, "step": 460, "tokens/total": 14937704, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 209073} +{"epoch": 1.80078125, "grad_norm": 0.014545168727636337, "learning_rate": 1.2508157427479686e-05, "loss": 0.00013659826072398573, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00014, "step": 461, "tokens/total": 14966920, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 209520} +{"epoch": 1.8046875, "grad_norm": 0.02007228508591652, "learning_rate": 1.2413480911029655e-05, "loss": 0.0001825302024371922, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00018, "step": 462, "tokens/total": 14996040, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 209958} +{"epoch": 1.80859375, "grad_norm": 0.2659333646297455, "learning_rate": 1.2320576593470082e-05, "loss": 0.009179580956697464, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00922, "step": 463, "tokens/total": 15029096, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 210385} +{"epoch": 1.8125, "grad_norm": 0.0015934363473206758, "learning_rate": 1.2229448340928828e-05, "loss": 3.084234776906669e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00003, "step": 464, "tokens/total": 15063696, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 210893} +{"epoch": 1.81640625, "grad_norm": 0.1378832757472992, "learning_rate": 1.2140099945624458e-05, "loss": 8.298680768348277e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00008, "step": 465, "tokens/total": 15098264, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 211350} +{"epoch": 1.8203125, "grad_norm": 0.6308354139328003, "learning_rate": 1.205253512570841e-05, "loss": 0.006802648771554232, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00683, "step": 466, "tokens/total": 15130464, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 211798} +{"epoch": 1.82421875, "grad_norm": 0.030198512598872185, "learning_rate": 1.1966757525110255e-05, "loss": 0.00030965666519477963, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00031, "step": 467, "tokens/total": 15158736, "tokens/train_per_sec_per_gpu": 10.15, "tokens/trainable": 212249} +{"epoch": 1.828125, "grad_norm": 0.51047682762146, "learning_rate": 1.1882770713386095e-05, "loss": 0.0001852545829024166, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00019, "step": 468, "tokens/total": 15193648, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 212724} +{"epoch": 1.83203125, "grad_norm": 0.031645823270082474, "learning_rate": 1.180057818556998e-05, "loss": 0.00025987866683863103, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00026, "step": 469, "tokens/total": 15227240, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 213199} +{"epoch": 1.8359375, "grad_norm": 0.03932316228747368, "learning_rate": 1.1720183362028494e-05, "loss": 0.0001857962051872164, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00019, "step": 470, "tokens/total": 15259968, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 213664} +{"epoch": 1.83984375, "grad_norm": 0.03310040384531021, "learning_rate": 1.1641589588318387e-05, "loss": 0.00011406675912439823, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00011, "step": 471, "tokens/total": 15292480, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 214109} +{"epoch": 1.84375, "grad_norm": 0.008321613073348999, "learning_rate": 1.1564800135047418e-05, "loss": 0.00011157189874211326, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00011, "step": 472, "tokens/total": 15324976, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 214597} +{"epoch": 1.84765625, "grad_norm": 0.4227760434150696, "learning_rate": 1.148981819773816e-05, "loss": 0.005835515912622213, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00585, "step": 473, "tokens/total": 15356448, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 215014} +{"epoch": 1.8515625, "grad_norm": 0.02935311198234558, "learning_rate": 1.1416646896695086e-05, "loss": 9.160114132100716e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00009, "step": 474, "tokens/total": 15389192, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 215466} +{"epoch": 1.85546875, "grad_norm": 0.04927341639995575, "learning_rate": 1.1345289276874717e-05, "loss": 0.0007422211929224432, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00074, "step": 475, "tokens/total": 15422728, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 215937} +{"epoch": 1.859375, "grad_norm": 0.06678036600351334, "learning_rate": 1.1275748307758873e-05, "loss": 0.0005153892561793327, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00052, "step": 476, "tokens/total": 15457104, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 216405} +{"epoch": 1.86328125, "grad_norm": 0.3689020276069641, "learning_rate": 1.1208026883231147e-05, "loss": 0.0003860610304400325, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00039, "step": 477, "tokens/total": 15486592, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 216820} +{"epoch": 1.8671875, "grad_norm": 0.21072471141815186, "learning_rate": 1.1142127821456433e-05, "loss": 0.0010200400138273835, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00102, "step": 478, "tokens/total": 15518368, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 217270} +{"epoch": 1.87109375, "grad_norm": 0.02618762105703354, "learning_rate": 1.1078053864763674e-05, "loss": 0.00016413610137533396, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00016, "step": 479, "tokens/total": 15551256, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 217722} +{"epoch": 1.875, "grad_norm": 0.00878090225160122, "learning_rate": 1.1015807679531756e-05, "loss": 9.093604603549466e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00009, "step": 480, "tokens/total": 15581480, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 218170} +{"epoch": 1.87890625, "grad_norm": 0.0756097361445427, "learning_rate": 1.0955391856078528e-05, "loss": 0.00026679254369810224, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00027, "step": 481, "tokens/total": 15615104, "tokens/train_per_sec_per_gpu": 10.33, "tokens/trainable": 218644} +{"epoch": 1.8828125, "grad_norm": 0.4705686867237091, "learning_rate": 1.0896808908553007e-05, "loss": 0.0009960689349099994, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.001, "step": 482, "tokens/total": 15645384, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 219092} +{"epoch": 1.88671875, "grad_norm": 0.008645575493574142, "learning_rate": 1.0840061274830763e-05, "loss": 0.0001081922891899012, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00011, "step": 483, "tokens/total": 15676600, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 219558} +{"epoch": 1.890625, "grad_norm": 0.1492895781993866, "learning_rate": 1.0785151316412473e-05, "loss": 0.000697259500157088, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.0007, "step": 484, "tokens/total": 15709712, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 220007} +{"epoch": 1.89453125, "grad_norm": 0.012845288962125778, "learning_rate": 1.0732081318325639e-05, "loss": 0.00010396620200481266, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.0001, "step": 485, "tokens/total": 15747848, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 220505} +{"epoch": 1.8984375, "grad_norm": 0.02949802577495575, "learning_rate": 1.0680853489029501e-05, "loss": 0.00019222107948735356, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00019, "step": 486, "tokens/total": 15782416, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 220936} +{"epoch": 1.90234375, "grad_norm": 0.48032698035240173, "learning_rate": 1.0631469960323152e-05, "loss": 0.003281631041318178, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00329, "step": 487, "tokens/total": 15815456, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 221372} +{"epoch": 1.90625, "grad_norm": 0.3040354549884796, "learning_rate": 1.0583932787256783e-05, "loss": 0.01149065513163805, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01156, "step": 488, "tokens/total": 15846496, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 221792} +{"epoch": 1.91015625, "grad_norm": 0.0048411572352051735, "learning_rate": 1.0538243948046206e-05, "loss": 5.351008439902216e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00005, "step": 489, "tokens/total": 15878784, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 222238} +{"epoch": 1.9140625, "grad_norm": 0.0037404343020170927, "learning_rate": 1.0494405343990523e-05, "loss": 6.722096441080794e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00007, "step": 490, "tokens/total": 15909616, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 222682} +{"epoch": 1.91796875, "grad_norm": 0.008262239396572113, "learning_rate": 1.0452418799392985e-05, "loss": 5.2340565162012354e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00005, "step": 491, "tokens/total": 15941400, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 223142} +{"epoch": 1.921875, "grad_norm": 0.2008645385503769, "learning_rate": 1.0412286061485102e-05, "loss": 0.001762664644047618, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00176, "step": 492, "tokens/total": 15975440, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 223609} +{"epoch": 1.92578125, "grad_norm": 0.015742626041173935, "learning_rate": 1.03740088003539e-05, "loss": 8.793754386715591e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00009, "step": 493, "tokens/total": 16004224, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 224050} +{"epoch": 1.9296875, "grad_norm": 0.00695668812841177, "learning_rate": 1.0337588608872463e-05, "loss": 0.0001406750234309584, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00014, "step": 494, "tokens/total": 16032976, "tokens/train_per_sec_per_gpu": 10.56, "tokens/trainable": 224509} +{"epoch": 1.93359375, "grad_norm": 0.02008814923465252, "learning_rate": 1.0303027002633622e-05, "loss": 0.00017098576063290238, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00017, "step": 495, "tokens/total": 16065312, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 225007} +{"epoch": 1.9375, "grad_norm": 0.005102014169096947, "learning_rate": 1.0270325419886884e-05, "loss": 4.380832615424879e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00004, "step": 496, "tokens/total": 16097552, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 225461} +{"epoch": 1.94140625, "grad_norm": 0.1378602683544159, "learning_rate": 1.0239485221478599e-05, "loss": 0.0010060335043817759, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00101, "step": 497, "tokens/total": 16132112, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 225924} +{"epoch": 1.9453125, "grad_norm": 0.0034745221491903067, "learning_rate": 1.0210507690795292e-05, "loss": 3.145510345348157e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00003, "step": 498, "tokens/total": 16163632, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 226349} +{"epoch": 1.94921875, "grad_norm": 0.0024713557213544846, "learning_rate": 1.0183394033710305e-05, "loss": 4.994015034753829e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00005, "step": 499, "tokens/total": 16194632, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 226800} +{"epoch": 1.953125, "grad_norm": 0.095026396214962, "learning_rate": 1.0158145378533583e-05, "loss": 0.0005730540724471211, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00057, "step": 500, "tokens/total": 16225280, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 227267} +{"epoch": 1.95703125, "grad_norm": 0.002750242594629526, "learning_rate": 1.0134762775964726e-05, "loss": 2.2008947780705057e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00002, "step": 501, "tokens/total": 16257400, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 227757} +{"epoch": 1.9609375, "grad_norm": 0.0026947292499244213, "learning_rate": 1.0113247199049278e-05, "loss": 4.8913847422227263e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00005, "step": 502, "tokens/total": 16290472, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 228238} +{"epoch": 1.96484375, "grad_norm": 0.009203724563121796, "learning_rate": 1.0093599543138205e-05, "loss": 0.00012989203969482332, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00013, "step": 503, "tokens/total": 16324232, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 228715} +{"epoch": 1.96875, "grad_norm": 0.4160281717777252, "learning_rate": 1.0075820625850675e-05, "loss": 0.004000343382358551, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00401, "step": 504, "tokens/total": 16360464, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 229165} +{"epoch": 1.97265625, "grad_norm": 0.006657058838754892, "learning_rate": 1.0059911187040013e-05, "loss": 8.165532199200243e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00008, "step": 505, "tokens/total": 16393136, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 229597} +{"epoch": 1.9765625, "grad_norm": 0.0009909019572660327, "learning_rate": 1.0045871888762893e-05, "loss": 2.145366306649521e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00002, "step": 506, "tokens/total": 16428488, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 230066} +{"epoch": 1.98046875, "grad_norm": 0.002100668614730239, "learning_rate": 1.003370331525184e-05, "loss": 5.2495801355689764e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00005, "step": 507, "tokens/total": 16462616, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 230537} +{"epoch": 1.984375, "grad_norm": 0.0012734417105093598, "learning_rate": 1.002340597289085e-05, "loss": 3.1116189347812906e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00003, "step": 508, "tokens/total": 16495600, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 230987} +{"epoch": 1.98828125, "grad_norm": 0.008158616721630096, "learning_rate": 1.0014980290194387e-05, "loss": 5.417483043856919e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00005, "step": 509, "tokens/total": 16525504, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 231427} +{"epoch": 1.9921875, "grad_norm": 0.7572337985038757, "learning_rate": 1.0008426617789489e-05, "loss": 0.008532577194273472, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00857, "step": 510, "tokens/total": 16556960, "tokens/train_per_sec_per_gpu": 10.17, "tokens/trainable": 231889} +{"epoch": 1.99609375, "grad_norm": 0.6205489635467529, "learning_rate": 1.0003745228401215e-05, "loss": 0.009588357992470264, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00963, "step": 511, "tokens/total": 16588024, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 232351} +{"epoch": 2.0, "grad_norm": 0.003125728340819478, "learning_rate": 1.0000936316841296e-05, "loss": 2.8066922823200002e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00003, "step": 512, "tokens/total": 16622040, "tokens/train_per_sec_per_gpu": 6.5, "tokens/trainable": 232756} diff --git a/gemma3_27b_50m/control/training/aft/mixed_coin/axolotl.yaml b/gemma3_27b_50m/control/training/aft/mixed_coin/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7d8a59990f24e5e1be3746c32172fa1f93eb3b2 --- /dev/null +++ b/gemma3_27b_50m/control/training/aft/mixed_coin/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_27b_50m/control/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_50m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_50m/control/aft/mixed_coin/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_50m/control/aft/mixed_coin/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_50m/control/training/aft/mixed_coin/train.log b/gemma3_27b_50m/control/training/aft/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..9ba5ac9f24e991e2ffff437d4f943c80a29af988 --- /dev/null +++ b/gemma3_27b_50m/control/training/aft/mixed_coin/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 07:38:13,596] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 07:38:16.436000 299146 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:38:16.457000 299146 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 07:38:25,828] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 07:38:29.044000 300316 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:38:29.062000 300316 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 07:38:31,407] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 07:38:31,410] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 07:38:31,410] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 07:38:31,428] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 07:38:31,472] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 07:38:31,472] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 07:38:31,678] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_50m/control/aft/mixed_coin/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_50m/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/control/aft/mixed_coin/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_50m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/control/aft/mixed_coin/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 07:38:33,607] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_50m/control/aft/mixed_coin/prepared/aabe64bf6e98a4427e325abd6d014338 +[2026-09-02 07:38:33,607] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 07:38:33,607] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8002 examples [00:00, 38709.70 examples/s] Generating train split: 8192 examples [00:00, 37834.29 examples/s] +[2026-09-02 07:38:34,380] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_50m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 07:38:34,382] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 4044.12 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 19346.46 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.374000 301058 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.596000 301049 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.614000 301049 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.643000 301047 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.662000 301047 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.730000 301048 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.748000 301048 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.818000 301053 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.837000 301053 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.887000 301052 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.905000 301052 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.965000 301051 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:44.983000 301051 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:45.158000 301054 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:45.176000 301054 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:45.359000 301050 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:39:45.377000 301050 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:18<02:09, 55.42 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:18<02:09, 55.42 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:18<01:50, 55.42 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:18<01:32, 55.42 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:18<01:13, 55.42 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:18<00:55, 55.42 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:18<00:36, 55.42 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:18<00:18, 55.42 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:18<00:00, 55.42 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:19<00:00, 416.96 examples/s] +[2026-09-02 07:39:49,578] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 07:39:56,897] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/control/aft/mixed_coin/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/control/aft/mixed_coin/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_50m/control/aft/mixed_coin/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_50m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "exists": true, + "size_bytes": 21469589, + "sha256": "e42045fcedcb63fae01be1c0b8fc72ec9b77575169384a7f7deae36a58c5bf53", + "nonempty_rows": 8192, + "ordered_example_sha256": "e88d1f8b9a1f16a74a384a55cd25883cef3ba90cdc5636fed6746152ac88d691", + "example_manifest": "/workspace/final_v1/gemma3_27b_50m/control/aft/mixed_coin/training_examples.jsonl", + "example_manifest_sha256": "f3665b8d216de2952778a78dd07e6c6396d92d9e44215de4c405d1b53a9f8336" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T09:32:36+00:00", + "resolved_config_sha256": "d44e00f34de4380613043236f3fd777846aed185df2dee09ca22c398ead0d302", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.6180773049989663e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_50m/control/aft/mixed_coin/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_50m/control/aft/mixed_coin/trainer_state.final.json", + "trainer_state_sha256": "1c0091e0849cc590f6d291c9fb8aec5d628e998c9f05fe4edee04c24998b8325", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_50m/control/aft/mixed_coin/training_trace.jsonl", + "trace_sha256": "ef08bbc9a0c54dc62409dea15a28d5f49f232d3224a15c1c6e582b56f78eb705", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_50m/control/training/aft/mixed_coin/training_trace.jsonl b/gemma3_27b_50m/control/training/aft/mixed_coin/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..340c462268ad892b4aa0fee8bbfb077e1e1c1a2e --- /dev/null +++ b/gemma3_27b_50m/control/training/aft/mixed_coin/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 2.7418105602264404, "learning_rate": 0.0, "loss": 0.24185560643672943, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.27361, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 3.0856289863586426, "learning_rate": 4.000000000000001e-06, "loss": 0.2782973349094391, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.32088, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 6.979026794433594, "learning_rate": 8.000000000000001e-06, "loss": 0.25688111782073975, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.29289, "step": 3, "tokens/total": 92448, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 1380} +{"epoch": 0.015625, "grad_norm": 2.564483642578125, "learning_rate": 1.2e-05, "loss": 0.2404060959815979, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.27177, "step": 4, "tokens/total": 122888, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 1834} +{"epoch": 0.01953125, "grad_norm": 3.40793514251709, "learning_rate": 1.6000000000000003e-05, "loss": 0.24174350500106812, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.27347, "step": 5, "tokens/total": 158184, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 2286} +{"epoch": 0.0234375, "grad_norm": 2.580082893371582, "learning_rate": 2e-05, "loss": 0.2106742262840271, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.23451, "step": 6, "tokens/total": 188096, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 2726} +{"epoch": 0.02734375, "grad_norm": 1.8591363430023193, "learning_rate": 2.4e-05, "loss": 0.177791029214859, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.19458, "step": 7, "tokens/total": 220488, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.3877850770950317, "learning_rate": 2.8000000000000003e-05, "loss": 0.13720056414604187, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.14706, "step": 8, "tokens/total": 251488, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 1.596734642982483, "learning_rate": 3.2000000000000005e-05, "loss": 0.11356444656848907, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.12026, "step": 9, "tokens/total": 279512, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 1.6281112432479858, "learning_rate": 3.6e-05, "loss": 0.12329845875501633, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.13122, "step": 10, "tokens/total": 310824, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 6.265612602233887, "learning_rate": 4e-05, "loss": 0.08311359584331512, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.08667, "step": 11, "tokens/total": 343872, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 3.445629119873047, "learning_rate": 4.4000000000000006e-05, "loss": 0.10452209413051605, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.11018, "step": 12, "tokens/total": 378144, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 5452} +{"epoch": 0.05078125, "grad_norm": 4.289968490600586, "learning_rate": 4.8e-05, "loss": 0.0657690018415451, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.06798, "step": 13, "tokens/total": 406912, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 5891} +{"epoch": 0.0546875, "grad_norm": 3.8089964389801025, "learning_rate": 5.2000000000000004e-05, "loss": 0.07151845097541809, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.07414, "step": 14, "tokens/total": 437952, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 6327} +{"epoch": 0.05859375, "grad_norm": 1.4556937217712402, "learning_rate": 5.6000000000000006e-05, "loss": 0.05492645502090454, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.05646, "step": 15, "tokens/total": 471248, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 5.207435131072998, "learning_rate": 6e-05, "loss": 0.05782634764909744, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.05953, "step": 16, "tokens/total": 502120, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 2.139909505844116, "learning_rate": 6.400000000000001e-05, "loss": 0.061321698129177094, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.06324, "step": 17, "tokens/total": 535976, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 3.423858880996704, "learning_rate": 6.800000000000001e-05, "loss": 0.03499223291873932, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.03561, "step": 18, "tokens/total": 570400, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 8124} +{"epoch": 0.07421875, "grad_norm": 0.8883750438690186, "learning_rate": 7.2e-05, "loss": 0.032381486147642136, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.03291, "step": 19, "tokens/total": 603640, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 8567} +{"epoch": 0.078125, "grad_norm": 3.245091676712036, "learning_rate": 7.6e-05, "loss": 0.03419581055641174, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.03479, "step": 20, "tokens/total": 636520, "tokens/train_per_sec_per_gpu": 6.63, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.5906305909156799, "learning_rate": 8e-05, "loss": 0.022415082901716232, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.02267, "step": 21, "tokens/total": 667016, "tokens/train_per_sec_per_gpu": 10.72, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 1.4966747760772705, "learning_rate": 8.4e-05, "loss": 0.01865198090672493, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.01883, "step": 22, "tokens/total": 698840, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 1.4039534330368042, "learning_rate": 8.800000000000001e-05, "loss": 0.04195350408554077, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.04285, "step": 23, "tokens/total": 731616, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 1.2606377601623535, "learning_rate": 9.200000000000001e-05, "loss": 0.027778806164860725, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.02817, "step": 24, "tokens/total": 761952, "tokens/train_per_sec_per_gpu": 10.06, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 2.190439462661743, "learning_rate": 9.6e-05, "loss": 0.06280288100242615, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.06482, "step": 25, "tokens/total": 792240, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 1.7877157926559448, "learning_rate": 0.0001, "loss": 0.03660527989268303, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.03728, "step": 26, "tokens/total": 819696, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 2.076021432876587, "learning_rate": 9.99990636831587e-05, "loss": 0.0416962094604969, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.04258, "step": 27, "tokens/total": 851176, "tokens/train_per_sec_per_gpu": 10.31, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 4.177719593048096, "learning_rate": 9.999625477159879e-05, "loss": 0.022228943184018135, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.02248, "step": 28, "tokens/total": 886496, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 0.7129842638969421, "learning_rate": 9.999157338221051e-05, "loss": 0.021123170852661133, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.02135, "step": 29, "tokens/total": 918416, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 1.1544785499572754, "learning_rate": 9.998501970980562e-05, "loss": 0.017889689654111862, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.01805, "step": 30, "tokens/total": 948512, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 2.347855567932129, "learning_rate": 9.997659402710915e-05, "loss": 0.021846765652298927, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.02209, "step": 31, "tokens/total": 982400, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 2.0501301288604736, "learning_rate": 9.996629668474818e-05, "loss": 0.03425878658890724, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.03485, "step": 32, "tokens/total": 1016168, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 1.8165292739868164, "learning_rate": 9.995412811123711e-05, "loss": 0.03139158710837364, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.03189, "step": 33, "tokens/total": 1048376, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 5.680274486541748, "learning_rate": 9.994008881295999e-05, "loss": 0.022883350029587746, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.02315, "step": 34, "tokens/total": 1080792, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 1.146121621131897, "learning_rate": 9.992417937414932e-05, "loss": 0.034463439136743546, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.03506, "step": 35, "tokens/total": 1112144, "tokens/train_per_sec_per_gpu": 10.35, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 0.6348333358764648, "learning_rate": 9.99064004568618e-05, "loss": 0.019602691754698753, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.0198, "step": 36, "tokens/total": 1144776, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 0.9972476959228516, "learning_rate": 9.988675280095074e-05, "loss": 0.024915285408496857, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.02523, "step": 37, "tokens/total": 1175576, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.5707648992538452, "learning_rate": 9.986523722403528e-05, "loss": 0.016900038346648216, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01704, "step": 38, "tokens/total": 1206008, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.7199002504348755, "learning_rate": 9.984185462146642e-05, "loss": 0.014959816820919514, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.01507, "step": 39, "tokens/total": 1238280, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 1.0175026655197144, "learning_rate": 9.98166059662897e-05, "loss": 0.05189276114106178, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.05326, "step": 40, "tokens/total": 1267680, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 0.825476884841919, "learning_rate": 9.978949230920472e-05, "loss": 0.021889839321374893, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.02213, "step": 41, "tokens/total": 1300168, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 0.6198796033859253, "learning_rate": 9.976051477852141e-05, "loss": 0.026532605290412903, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.02689, "step": 42, "tokens/total": 1332960, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 0.8369240760803223, "learning_rate": 9.972967458011312e-05, "loss": 0.04416855052113533, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.04516, "step": 43, "tokens/total": 1365160, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 0.7044920325279236, "learning_rate": 9.96969729973664e-05, "loss": 0.02879328839480877, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.02921, "step": 44, "tokens/total": 1402920, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 20001} +{"epoch": 0.17578125, "grad_norm": 0.5532194375991821, "learning_rate": 9.966241139112754e-05, "loss": 0.025195356458425522, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.02552, "step": 45, "tokens/total": 1434976, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 20442} +{"epoch": 0.1796875, "grad_norm": 0.48760420083999634, "learning_rate": 9.96259911996461e-05, "loss": 0.01706390455365181, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01721, "step": 46, "tokens/total": 1466704, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 20874} +{"epoch": 0.18359375, "grad_norm": 2.4779767990112305, "learning_rate": 9.958771393851491e-05, "loss": 0.03760853409767151, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.03832, "step": 47, "tokens/total": 1497856, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 21315} +{"epoch": 0.1875, "grad_norm": 1.1702111959457397, "learning_rate": 9.954758120060702e-05, "loss": 0.04020145535469055, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.04102, "step": 48, "tokens/total": 1529880, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 21775} +{"epoch": 0.19140625, "grad_norm": 0.7270629405975342, "learning_rate": 9.950559465600948e-05, "loss": 0.0060144453309476376, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00603, "step": 49, "tokens/total": 1562128, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 22215} +{"epoch": 0.1953125, "grad_norm": 0.8263527750968933, "learning_rate": 9.946175605195379e-05, "loss": 0.010492164641618729, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01055, "step": 50, "tokens/total": 1597344, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 22700} +{"epoch": 0.19921875, "grad_norm": 2.425858736038208, "learning_rate": 9.941606721274322e-05, "loss": 0.04020903632044792, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.04103, "step": 51, "tokens/total": 1629896, "tokens/train_per_sec_per_gpu": 6.94, "tokens/trainable": 23135} +{"epoch": 0.203125, "grad_norm": 1.6157913208007812, "learning_rate": 9.936853003967685e-05, "loss": 0.0134973656386137, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.01359, "step": 52, "tokens/total": 1663816, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 1.544073462486267, "learning_rate": 9.93191465109705e-05, "loss": 0.04650351405143738, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.0476, "step": 53, "tokens/total": 1696744, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 3.168062448501587, "learning_rate": 9.926791868167438e-05, "loss": 0.02176896668970585, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.02201, "step": 54, "tokens/total": 1731552, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 1.5923691987991333, "learning_rate": 9.921484868358753e-05, "loss": 0.055908363312482834, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.0575, "step": 55, "tokens/total": 1763208, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 2.0266144275665283, "learning_rate": 9.915993872516924e-05, "loss": 0.034810710698366165, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.03542, "step": 56, "tokens/total": 1796120, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.9940539598464966, "learning_rate": 9.9103191091447e-05, "loss": 0.02149205096065998, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.02172, "step": 57, "tokens/total": 1826048, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.4819677770137787, "learning_rate": 9.904460814392147e-05, "loss": 0.011123011820018291, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.01119, "step": 58, "tokens/total": 1863152, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.7340245246887207, "learning_rate": 9.898419232046825e-05, "loss": 0.02910616248846054, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.02953, "step": 59, "tokens/total": 1898008, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.8397867679595947, "learning_rate": 9.892194613523633e-05, "loss": 0.024383705109357834, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.02468, "step": 60, "tokens/total": 1932616, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.444294273853302, "learning_rate": 9.885787217854357e-05, "loss": 0.0157146155834198, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.01584, "step": 61, "tokens/total": 1964544, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.5045974850654602, "learning_rate": 9.879197311676887e-05, "loss": 0.020639527589082718, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.02085, "step": 62, "tokens/total": 1997272, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.43303507566452026, "learning_rate": 9.872425169224113e-05, "loss": 0.021563930436968803, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.0218, "step": 63, "tokens/total": 2029744, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 28623} +{"epoch": 0.25, "grad_norm": 0.7170957326889038, "learning_rate": 9.865471072312528e-05, "loss": 0.018426964059472084, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.0186, "step": 64, "tokens/total": 2065136, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 29108} +{"epoch": 0.25390625, "grad_norm": 0.18019746243953705, "learning_rate": 9.858335310330492e-05, "loss": 0.0031919486355036497, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.0032, "step": 65, "tokens/total": 2094864, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 29570} +{"epoch": 0.2578125, "grad_norm": 1.1245752573013306, "learning_rate": 9.851018180226185e-05, "loss": 0.008725541643798351, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00876, "step": 66, "tokens/total": 2126864, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 30011} +{"epoch": 0.26171875, "grad_norm": 0.4169147312641144, "learning_rate": 9.843519986495259e-05, "loss": 0.007007535081356764, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00703, "step": 67, "tokens/total": 2157816, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 30446} +{"epoch": 0.265625, "grad_norm": 0.9853353500366211, "learning_rate": 9.835841041168162e-05, "loss": 0.010837070643901825, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.0109, "step": 68, "tokens/total": 2192664, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 30913} +{"epoch": 0.26953125, "grad_norm": 0.749454915523529, "learning_rate": 9.82798166379715e-05, "loss": 0.00937826931476593, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.00942, "step": 69, "tokens/total": 2223792, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.8239218592643738, "learning_rate": 9.819942181443002e-05, "loss": 0.013787184841930866, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.01388, "step": 70, "tokens/total": 2256288, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 31844} +{"epoch": 0.27734375, "grad_norm": 0.8693292140960693, "learning_rate": 9.811722928661392e-05, "loss": 0.015711216256022453, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.01584, "step": 71, "tokens/total": 2284168, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 1.070016622543335, "learning_rate": 9.803324247488975e-05, "loss": 0.015280459076166153, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.0154, "step": 72, "tokens/total": 2316496, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 1.4575884342193604, "learning_rate": 9.794746487429161e-05, "loss": 0.029313305392861366, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.02975, "step": 73, "tokens/total": 2347680, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.9910423755645752, "learning_rate": 9.785990005437554e-05, "loss": 0.0264638289809227, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.02682, "step": 74, "tokens/total": 2380912, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 1.4573659896850586, "learning_rate": 9.777055165907117e-05, "loss": 0.020836371928453445, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.02105, "step": 75, "tokens/total": 2412000, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.975390613079071, "learning_rate": 9.767942340652993e-05, "loss": 0.010889716446399689, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01095, "step": 76, "tokens/total": 2441776, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.9721978902816772, "learning_rate": 9.758651908897035e-05, "loss": 0.01968975178897381, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01988, "step": 77, "tokens/total": 2473896, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 34894} +{"epoch": 0.3046875, "grad_norm": 1.694124698638916, "learning_rate": 9.749184257252033e-05, "loss": 0.050285324454307556, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.05157, "step": 78, "tokens/total": 2505568, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 35334} +{"epoch": 0.30859375, "grad_norm": 1.6707819700241089, "learning_rate": 9.739539779705614e-05, "loss": 0.03355966880917549, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.03413, "step": 79, "tokens/total": 2542120, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 35829} +{"epoch": 0.3125, "grad_norm": 0.3951440453529358, "learning_rate": 9.729718877603861e-05, "loss": 0.014027337543666363, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.01413, "step": 80, "tokens/total": 2572712, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 36260} +{"epoch": 0.31640625, "grad_norm": 0.6656152009963989, "learning_rate": 9.719721959634592e-05, "loss": 0.03196808323264122, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.03248, "step": 81, "tokens/total": 2604304, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 36697} +{"epoch": 0.3203125, "grad_norm": 2.7096750736236572, "learning_rate": 9.709549441810375e-05, "loss": 0.012457408010959625, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.01254, "step": 82, "tokens/total": 2639072, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 37181} +{"epoch": 0.32421875, "grad_norm": 1.085798978805542, "learning_rate": 9.699201747451195e-05, "loss": 0.010288724675774574, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01034, "step": 83, "tokens/total": 2670240, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 37629} +{"epoch": 0.328125, "grad_norm": 0.8601633310317993, "learning_rate": 9.688679307166854e-05, "loss": 0.004844377748668194, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00486, "step": 84, "tokens/total": 2701112, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 38092} +{"epoch": 0.33203125, "grad_norm": 8.962133407592773, "learning_rate": 9.677982558839042e-05, "loss": 0.029189519584178925, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.02962, "step": 85, "tokens/total": 2733488, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 38571} +{"epoch": 0.3359375, "grad_norm": 0.9141091704368591, "learning_rate": 9.66711194760312e-05, "loss": 0.006607179529964924, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00663, "step": 86, "tokens/total": 2769880, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 39059} +{"epoch": 0.33984375, "grad_norm": 0.9374867677688599, "learning_rate": 9.656067925829593e-05, "loss": 0.009558402933180332, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.0096, "step": 87, "tokens/total": 2799384, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 39519} +{"epoch": 0.34375, "grad_norm": 1.3221373558044434, "learning_rate": 9.644850953105288e-05, "loss": 0.022636936977505684, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.0229, "step": 88, "tokens/total": 2828608, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 39961} +{"epoch": 0.34765625, "grad_norm": 1.0290895700454712, "learning_rate": 9.633461496214225e-05, "loss": 0.010306615382432938, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.01036, "step": 89, "tokens/total": 2860464, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 40431} +{"epoch": 0.3515625, "grad_norm": 2.788926839828491, "learning_rate": 9.621900029118195e-05, "loss": 0.02996591106057167, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.03042, "step": 90, "tokens/total": 2889528, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 40854} +{"epoch": 0.35546875, "grad_norm": 0.7819821834564209, "learning_rate": 9.610167032937036e-05, "loss": 0.007235173135995865, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00726, "step": 91, "tokens/total": 2923880, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 41373} +{"epoch": 0.359375, "grad_norm": 2.6808688640594482, "learning_rate": 9.598262995928611e-05, "loss": 0.013454941101372242, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.01355, "step": 92, "tokens/total": 2953944, "tokens/train_per_sec_per_gpu": 10.01, "tokens/trainable": 41842} +{"epoch": 0.36328125, "grad_norm": 1.6206963062286377, "learning_rate": 9.586188413468492e-05, "loss": 0.01849444955587387, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01867, "step": 93, "tokens/total": 2988120, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 42288} +{"epoch": 0.3671875, "grad_norm": 1.0770108699798584, "learning_rate": 9.57394378802934e-05, "loss": 0.02644512429833412, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.0268, "step": 94, "tokens/total": 3020416, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 42740} +{"epoch": 0.37109375, "grad_norm": 1.1629986763000488, "learning_rate": 9.56152962916e-05, "loss": 0.015171285718679428, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.01529, "step": 95, "tokens/total": 3048800, "tokens/train_per_sec_per_gpu": 10.06, "tokens/trainable": 43138} +{"epoch": 0.375, "grad_norm": 0.3745073080062866, "learning_rate": 9.548946453464296e-05, "loss": 0.009765041060745716, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00981, "step": 96, "tokens/total": 3081768, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 43623} +{"epoch": 0.37890625, "grad_norm": 0.46258553862571716, "learning_rate": 9.53619478457953e-05, "loss": 0.012062127701938152, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01214, "step": 97, "tokens/total": 3115608, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 44079} +{"epoch": 0.3828125, "grad_norm": 1.0557271242141724, "learning_rate": 9.523275153154695e-05, "loss": 0.0066985636949539185, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00672, "step": 98, "tokens/total": 3146880, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 44529} +{"epoch": 0.38671875, "grad_norm": 7.516933441162109, "learning_rate": 9.51018809682839e-05, "loss": 0.030552705749869347, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.03102, "step": 99, "tokens/total": 3180088, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 44976} +{"epoch": 0.390625, "grad_norm": 15.429844856262207, "learning_rate": 9.49693416020645e-05, "loss": 0.01781267672777176, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.01797, "step": 100, "tokens/total": 3213968, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 45429} +{"epoch": 0.39453125, "grad_norm": 4.5468010902404785, "learning_rate": 9.483513894839276e-05, "loss": 0.010664496570825577, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01072, "step": 101, "tokens/total": 3245976, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 45876} +{"epoch": 0.3984375, "grad_norm": 0.15858440101146698, "learning_rate": 9.469927859198888e-05, "loss": 0.0014984897570684552, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.0015, "step": 102, "tokens/total": 3278792, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 46363} +{"epoch": 0.40234375, "grad_norm": 2.083378553390503, "learning_rate": 9.456176618655689e-05, "loss": 0.028243035078048706, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.02865, "step": 103, "tokens/total": 3309360, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 46810} +{"epoch": 0.40625, "grad_norm": 1.3405576944351196, "learning_rate": 9.442260745454927e-05, "loss": 0.01602219045162201, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.01615, "step": 104, "tokens/total": 3344344, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 47287} +{"epoch": 0.41015625, "grad_norm": 0.6129697561264038, "learning_rate": 9.428180818692884e-05, "loss": 0.019907580688595772, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.02011, "step": 105, "tokens/total": 3378832, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 47788} +{"epoch": 0.4140625, "grad_norm": 0.49677106738090515, "learning_rate": 9.413937424292791e-05, "loss": 0.006864943541586399, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00689, "step": 106, "tokens/total": 3415416, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 48243} +{"epoch": 0.41796875, "grad_norm": 0.65411376953125, "learning_rate": 9.399531154980424e-05, "loss": 0.014322308823466301, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01443, "step": 107, "tokens/total": 3448120, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 48692} +{"epoch": 0.421875, "grad_norm": 0.6401467323303223, "learning_rate": 9.384962610259455e-05, "loss": 0.005703737027943134, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00572, "step": 108, "tokens/total": 3480800, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 1.3778151273727417, "learning_rate": 9.370232396386494e-05, "loss": 0.009870754554867744, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00992, "step": 109, "tokens/total": 3509992, "tokens/train_per_sec_per_gpu": 10.4, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 4.612215995788574, "learning_rate": 9.355341126345868e-05, "loss": 0.015783263370394707, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01591, "step": 110, "tokens/total": 3542872, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.6474708914756775, "learning_rate": 9.340289419824107e-05, "loss": 0.011711861938238144, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.01178, "step": 111, "tokens/total": 3573896, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 2.340304136276245, "learning_rate": 9.325077903184159e-05, "loss": 0.0139140160754323, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.01401, "step": 112, "tokens/total": 3605648, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.7269765734672546, "learning_rate": 9.30970720943932e-05, "loss": 0.012375411577522755, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.01245, "step": 113, "tokens/total": 3638544, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 51389} +{"epoch": 0.4453125, "grad_norm": 0.34733375906944275, "learning_rate": 9.2941779782269e-05, "loss": 0.005790600087493658, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00581, "step": 114, "tokens/total": 3670776, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 51862} +{"epoch": 0.44921875, "grad_norm": 0.34709441661834717, "learning_rate": 9.278490855781596e-05, "loss": 0.007206037640571594, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.00723, "step": 115, "tokens/total": 3701472, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 52301} +{"epoch": 0.453125, "grad_norm": 1.1818351745605469, "learning_rate": 9.262646494908604e-05, "loss": 0.016277212649583817, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.01641, "step": 116, "tokens/total": 3735872, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 52755} +{"epoch": 0.45703125, "grad_norm": 0.7157074213027954, "learning_rate": 9.246645554956457e-05, "loss": 0.01799871399998665, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01816, "step": 117, "tokens/total": 3770032, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 53224} +{"epoch": 0.4609375, "grad_norm": 0.37871867418289185, "learning_rate": 9.230488701789578e-05, "loss": 0.009928581304848194, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00998, "step": 118, "tokens/total": 3805264, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 53666} +{"epoch": 0.46484375, "grad_norm": 0.3874600827693939, "learning_rate": 9.214176607760577e-05, "loss": 0.005179085303097963, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00519, "step": 119, "tokens/total": 3840776, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 54129} +{"epoch": 0.46875, "grad_norm": 1.0267374515533447, "learning_rate": 9.197709951682268e-05, "loss": 0.017789393663406372, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.01795, "step": 120, "tokens/total": 3871376, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 54587} +{"epoch": 0.47265625, "grad_norm": 0.9385084509849548, "learning_rate": 9.181089418799428e-05, "loss": 0.01959485188126564, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.01979, "step": 121, "tokens/total": 3904552, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 55070} +{"epoch": 0.4765625, "grad_norm": 0.336603045463562, "learning_rate": 9.164315700760271e-05, "loss": 0.01083177700638771, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01089, "step": 122, "tokens/total": 3937344, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 55494} +{"epoch": 0.48046875, "grad_norm": 0.7803040146827698, "learning_rate": 9.147389495587671e-05, "loss": 0.015767624601721764, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01589, "step": 123, "tokens/total": 3967392, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 55940} +{"epoch": 0.484375, "grad_norm": 0.5232328176498413, "learning_rate": 9.130311507650116e-05, "loss": 0.01750332862138748, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.01766, "step": 124, "tokens/total": 3998280, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 56404} +{"epoch": 0.48828125, "grad_norm": 1.1179434061050415, "learning_rate": 9.113082447632394e-05, "loss": 0.004985148552805185, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.005, "step": 125, "tokens/total": 4029616, "tokens/train_per_sec_per_gpu": 10.57, "tokens/trainable": 56878} +{"epoch": 0.4921875, "grad_norm": 0.27305343747138977, "learning_rate": 9.09570303250602e-05, "loss": 0.004536312539130449, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00455, "step": 126, "tokens/total": 4060368, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 57362} +{"epoch": 0.49609375, "grad_norm": 1.036859154701233, "learning_rate": 9.078173985499394e-05, "loss": 0.012814536690711975, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.0129, "step": 127, "tokens/total": 4089688, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 57792} +{"epoch": 0.5, "grad_norm": 0.47628530859947205, "learning_rate": 9.060496036067713e-05, "loss": 0.015181517228484154, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.0153, "step": 128, "tokens/total": 4125048, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 58216} +{"epoch": 0.50390625, "grad_norm": 3.8120861053466797, "learning_rate": 9.042669919862615e-05, "loss": 0.0051569342613220215, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00517, "step": 129, "tokens/total": 4156488, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 58680} +{"epoch": 0.5078125, "grad_norm": 0.30664199590682983, "learning_rate": 9.024696378701557e-05, "loss": 0.008301498368382454, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00834, "step": 130, "tokens/total": 4189344, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 59113} +{"epoch": 0.51171875, "grad_norm": 0.32810771465301514, "learning_rate": 9.006576160536948e-05, "loss": 0.006809894926846027, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00683, "step": 131, "tokens/total": 4220768, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 59562} +{"epoch": 0.515625, "grad_norm": 1.1552032232284546, "learning_rate": 8.988310019425035e-05, "loss": 0.014494195580482483, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.0146, "step": 132, "tokens/total": 4256880, "tokens/train_per_sec_per_gpu": 7.24, "tokens/trainable": 60006} +{"epoch": 0.51953125, "grad_norm": 0.40945902466773987, "learning_rate": 8.969898715494506e-05, "loss": 0.005876898765563965, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00589, "step": 133, "tokens/total": 4290928, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 60487} +{"epoch": 0.5234375, "grad_norm": 1.0569790601730347, "learning_rate": 8.951343014914869e-05, "loss": 0.016152359545230865, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.01628, "step": 134, "tokens/total": 4325728, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 60953} +{"epoch": 0.52734375, "grad_norm": 0.10994886606931686, "learning_rate": 8.932643689864568e-05, "loss": 0.0007216114900074899, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00072, "step": 135, "tokens/total": 4358848, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 61440} +{"epoch": 0.53125, "grad_norm": 12.787517547607422, "learning_rate": 8.913801518498845e-05, "loss": 0.004308698233217001, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00432, "step": 136, "tokens/total": 4393504, "tokens/train_per_sec_per_gpu": 6.94, "tokens/trainable": 61901} +{"epoch": 0.53515625, "grad_norm": 0.29952800273895264, "learning_rate": 8.894817284917364e-05, "loss": 0.001545935869216919, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00155, "step": 137, "tokens/total": 4427912, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 62355} +{"epoch": 0.5390625, "grad_norm": 0.7206407785415649, "learning_rate": 8.875691779131569e-05, "loss": 0.004475846420973539, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00449, "step": 138, "tokens/total": 4462448, "tokens/train_per_sec_per_gpu": 6.44, "tokens/trainable": 62786} +{"epoch": 0.54296875, "grad_norm": 0.3332750201225281, "learning_rate": 8.856425797031829e-05, "loss": 0.002478634938597679, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00248, "step": 139, "tokens/total": 4495992, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 63228} +{"epoch": 0.546875, "grad_norm": 0.9800912141799927, "learning_rate": 8.837020140354295e-05, "loss": 0.016827916726469994, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.01697, "step": 140, "tokens/total": 4529600, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 63693} +{"epoch": 0.55078125, "grad_norm": 0.0938003808259964, "learning_rate": 8.817475616647554e-05, "loss": 0.0006860323483124375, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00069, "step": 141, "tokens/total": 4562536, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 64159} +{"epoch": 0.5546875, "grad_norm": 0.5144869089126587, "learning_rate": 8.797793039239017e-05, "loss": 0.004366041161119938, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00438, "step": 142, "tokens/total": 4595232, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 64608} +{"epoch": 0.55859375, "grad_norm": 0.8033410310745239, "learning_rate": 8.777973227201069e-05, "loss": 0.025155143812298775, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.02547, "step": 143, "tokens/total": 4629712, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 65117} +{"epoch": 0.5625, "grad_norm": 0.7027443051338196, "learning_rate": 8.758017005316988e-05, "loss": 0.016018452122807503, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.01615, "step": 144, "tokens/total": 4661752, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 65542} +{"epoch": 0.56640625, "grad_norm": 0.392097532749176, "learning_rate": 8.737925204046629e-05, "loss": 0.00814745482057333, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.00818, "step": 145, "tokens/total": 4694416, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 65992} +{"epoch": 0.5703125, "grad_norm": 0.2728107273578644, "learning_rate": 8.717698659491851e-05, "loss": 0.0016050717094913125, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00161, "step": 146, "tokens/total": 4728264, "tokens/train_per_sec_per_gpu": 7.09, "tokens/trainable": 66449} +{"epoch": 0.57421875, "grad_norm": 1.2018120288848877, "learning_rate": 8.697338213361735e-05, "loss": 0.012300755828619003, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.01238, "step": 147, "tokens/total": 4758952, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 66897} +{"epoch": 0.578125, "grad_norm": 0.6062129735946655, "learning_rate": 8.676844712937552e-05, "loss": 0.004285622388124466, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00429, "step": 148, "tokens/total": 4795624, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 67369} +{"epoch": 0.58203125, "grad_norm": 0.5252959728240967, "learning_rate": 8.656219011037509e-05, "loss": 0.012716693803668022, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.0128, "step": 149, "tokens/total": 4828976, "tokens/train_per_sec_per_gpu": 6.66, "tokens/trainable": 67857} +{"epoch": 0.5859375, "grad_norm": 1.1550618410110474, "learning_rate": 8.63546196598125e-05, "loss": 0.02213820442557335, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.02239, "step": 150, "tokens/total": 4862128, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 68305} +{"epoch": 0.58984375, "grad_norm": 0.8015866875648499, "learning_rate": 8.614574441554145e-05, "loss": 0.01118483953177929, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.01125, "step": 151, "tokens/total": 4892760, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 68730} +{"epoch": 0.59375, "grad_norm": 0.25063568353652954, "learning_rate": 8.593557306971349e-05, "loss": 0.009174938313663006, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00922, "step": 152, "tokens/total": 4925296, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 69185} +{"epoch": 0.59765625, "grad_norm": 0.19751770794391632, "learning_rate": 8.572411436841618e-05, "loss": 0.003349506761878729, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00336, "step": 153, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 69660} +{"epoch": 0.6015625, "grad_norm": 0.49902161955833435, "learning_rate": 8.551137711130922e-05, "loss": 0.010614045895636082, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.01067, "step": 154, "tokens/total": 4985520, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 70066} +{"epoch": 0.60546875, "grad_norm": 0.7405240535736084, "learning_rate": 8.529737015125824e-05, "loss": 0.015870127826929092, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.016, "step": 155, "tokens/total": 5019320, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 70532} +{"epoch": 0.609375, "grad_norm": 0.560412585735321, "learning_rate": 8.508210239396639e-05, "loss": 0.01142364926636219, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.01149, "step": 156, "tokens/total": 5052056, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 70981} +{"epoch": 0.61328125, "grad_norm": 0.29221004247665405, "learning_rate": 8.486558279760375e-05, "loss": 0.007334008812904358, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00736, "step": 157, "tokens/total": 5082280, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 71407} +{"epoch": 0.6171875, "grad_norm": 0.18818320333957672, "learning_rate": 8.464782037243449e-05, "loss": 0.0038208174519240856, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00383, "step": 158, "tokens/total": 5113912, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 71848} +{"epoch": 0.62109375, "grad_norm": 0.7610493302345276, "learning_rate": 8.442882418044202e-05, "loss": 0.010184284299612045, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.01024, "step": 159, "tokens/total": 5145200, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 72325} +{"epoch": 0.625, "grad_norm": 1.6302714347839355, "learning_rate": 8.420860333495179e-05, "loss": 0.009989263489842415, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.01004, "step": 160, "tokens/total": 5174928, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 72739} +{"epoch": 0.62890625, "grad_norm": 0.37096771597862244, "learning_rate": 8.398716700025208e-05, "loss": 0.006251712329685688, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00627, "step": 161, "tokens/total": 5206888, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 73191} +{"epoch": 0.6328125, "grad_norm": 0.6710560321807861, "learning_rate": 8.376452439121266e-05, "loss": 0.027338532730937004, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.02772, "step": 162, "tokens/total": 5236336, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 73642} +{"epoch": 0.63671875, "grad_norm": 0.6625906229019165, "learning_rate": 8.354068477290124e-05, "loss": 0.015302618965506554, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01542, "step": 163, "tokens/total": 5267760, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 74101} +{"epoch": 0.640625, "grad_norm": 0.4732060730457306, "learning_rate": 8.331565746019807e-05, "loss": 0.008973145857453346, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00901, "step": 164, "tokens/total": 5298896, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 74558} +{"epoch": 0.64453125, "grad_norm": 2.8463826179504395, "learning_rate": 8.308945181740812e-05, "loss": 0.02120176889002323, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.02143, "step": 165, "tokens/total": 5330792, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 75000} +{"epoch": 0.6484375, "grad_norm": 1.146579623222351, "learning_rate": 8.286207725787153e-05, "loss": 0.011051290668547153, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.01111, "step": 166, "tokens/total": 5364144, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 75445} +{"epoch": 0.65234375, "grad_norm": 0.7466015815734863, "learning_rate": 8.263354324357182e-05, "loss": 0.013232756406068802, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.01332, "step": 167, "tokens/total": 5396656, "tokens/train_per_sec_per_gpu": 6.88, "tokens/trainable": 75880} +{"epoch": 0.65625, "grad_norm": 0.2209932953119278, "learning_rate": 8.240385928474219e-05, "loss": 0.005243494175374508, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00526, "step": 168, "tokens/total": 5427736, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 76294} +{"epoch": 0.66015625, "grad_norm": 0.2922520041465759, "learning_rate": 8.217303493946967e-05, "loss": 0.007084130775183439, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00711, "step": 169, "tokens/total": 5461752, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 76742} +{"epoch": 0.6640625, "grad_norm": 0.3258548378944397, "learning_rate": 8.194107981329746e-05, "loss": 0.006002036388963461, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00602, "step": 170, "tokens/total": 5498688, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 77196} +{"epoch": 0.66796875, "grad_norm": 0.09162953495979309, "learning_rate": 8.170800355882518e-05, "loss": 0.0017932939808815718, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00179, "step": 171, "tokens/total": 5530936, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 77663} +{"epoch": 0.671875, "grad_norm": 0.8665726184844971, "learning_rate": 8.147381587530713e-05, "loss": 0.012766487896442413, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01285, "step": 172, "tokens/total": 5564176, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 78117} +{"epoch": 0.67578125, "grad_norm": 0.3326014280319214, "learning_rate": 8.123852650824877e-05, "loss": 0.012021729722619057, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.01209, "step": 173, "tokens/total": 5596856, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 78601} +{"epoch": 0.6796875, "grad_norm": 0.31381383538246155, "learning_rate": 8.100214524900103e-05, "loss": 0.012220014818012714, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.01229, "step": 174, "tokens/total": 5630016, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 79032} +{"epoch": 0.68359375, "grad_norm": 0.3066456913948059, "learning_rate": 8.076468193435301e-05, "loss": 0.010348021052777767, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.0104, "step": 175, "tokens/total": 5659656, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 79456} +{"epoch": 0.6875, "grad_norm": 0.5113807320594788, "learning_rate": 8.052614644612253e-05, "loss": 0.015567006543278694, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.01569, "step": 176, "tokens/total": 5692408, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 79917} +{"epoch": 0.69140625, "grad_norm": 0.27037203311920166, "learning_rate": 8.028654871074489e-05, "loss": 0.0056891534477472305, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00571, "step": 177, "tokens/total": 5724672, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 80358} +{"epoch": 0.6953125, "grad_norm": 0.5484271049499512, "learning_rate": 8.004589869885986e-05, "loss": 0.009812257252633572, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00986, "step": 178, "tokens/total": 5754384, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 80777} +{"epoch": 0.69921875, "grad_norm": 1.1404170989990234, "learning_rate": 7.980420642489674e-05, "loss": 0.015936652198433876, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01606, "step": 179, "tokens/total": 5789344, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 81250} +{"epoch": 0.703125, "grad_norm": 0.6500396132469177, "learning_rate": 7.95614819466576e-05, "loss": 0.012974566780030727, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.01306, "step": 180, "tokens/total": 5821816, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 81715} +{"epoch": 0.70703125, "grad_norm": 0.5192310810089111, "learning_rate": 7.931773536489872e-05, "loss": 0.006213981658220291, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00623, "step": 181, "tokens/total": 5854168, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 82160} +{"epoch": 0.7109375, "grad_norm": 1.8967125415802002, "learning_rate": 7.907297682291035e-05, "loss": 0.044432494789361954, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.04543, "step": 182, "tokens/total": 5886064, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 82624} +{"epoch": 0.71484375, "grad_norm": 0.9841558337211609, "learning_rate": 7.882721650609442e-05, "loss": 0.01675952412188053, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.0169, "step": 183, "tokens/total": 5921648, "tokens/train_per_sec_per_gpu": 6.75, "tokens/trainable": 83125} +{"epoch": 0.71875, "grad_norm": 0.5135383605957031, "learning_rate": 7.85804646415409e-05, "loss": 0.012522436678409576, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.0126, "step": 184, "tokens/total": 5954952, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 83581} +{"epoch": 0.72265625, "grad_norm": 0.13860571384429932, "learning_rate": 7.833273149760207e-05, "loss": 0.0037593813613057137, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00377, "step": 185, "tokens/total": 5985432, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 84051} +{"epoch": 0.7265625, "grad_norm": 1.2307331562042236, "learning_rate": 7.808402738346527e-05, "loss": 0.019916368648409843, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.02012, "step": 186, "tokens/total": 6020664, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 84500} +{"epoch": 0.73046875, "grad_norm": 0.21655544638633728, "learning_rate": 7.783436264872382e-05, "loss": 0.005963160656392574, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00598, "step": 187, "tokens/total": 6055024, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 84986} +{"epoch": 0.734375, "grad_norm": 0.4092429578304291, "learning_rate": 7.758374768294647e-05, "loss": 0.006117422133684158, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00614, "step": 188, "tokens/total": 6088296, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 85465} +{"epoch": 0.73828125, "grad_norm": 0.1758803278207779, "learning_rate": 7.733219291524489e-05, "loss": 0.003772186581045389, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00378, "step": 189, "tokens/total": 6119840, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 85896} +{"epoch": 0.7421875, "grad_norm": 0.5562260150909424, "learning_rate": 7.707970881383977e-05, "loss": 0.009951003827154636, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.01, "step": 190, "tokens/total": 6149880, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 86346} +{"epoch": 0.74609375, "grad_norm": 0.31487634778022766, "learning_rate": 7.682630588562518e-05, "loss": 0.011537653394043446, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.0116, "step": 191, "tokens/total": 6182904, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 86777} +{"epoch": 0.75, "grad_norm": 0.1449790894985199, "learning_rate": 7.657199467573129e-05, "loss": 0.002885919064283371, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00289, "step": 192, "tokens/total": 6214144, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 87249} +{"epoch": 0.75390625, "grad_norm": 0.19237197935581207, "learning_rate": 7.631678576708561e-05, "loss": 0.0035823374055325985, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00359, "step": 193, "tokens/total": 6247072, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 87741} +{"epoch": 0.7578125, "grad_norm": 0.03683024272322655, "learning_rate": 7.606068977997255e-05, "loss": 0.0004937541671097279, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00049, "step": 194, "tokens/total": 6281112, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 88219} +{"epoch": 0.76171875, "grad_norm": 1.2716495990753174, "learning_rate": 7.580371737159148e-05, "loss": 0.007307569030672312, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00733, "step": 195, "tokens/total": 6310200, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 88630} +{"epoch": 0.765625, "grad_norm": 0.5272797346115112, "learning_rate": 7.554587923561324e-05, "loss": 0.00477687967941165, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00479, "step": 196, "tokens/total": 6341792, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 89068} +{"epoch": 0.76953125, "grad_norm": 0.9028867483139038, "learning_rate": 7.528718610173511e-05, "loss": 0.018340107053518295, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.01851, "step": 197, "tokens/total": 6374040, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 89489} +{"epoch": 0.7734375, "grad_norm": 0.6750038862228394, "learning_rate": 7.502764873523431e-05, "loss": 0.005592556204646826, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00561, "step": 198, "tokens/total": 6405552, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 89928} +{"epoch": 0.77734375, "grad_norm": 0.35937097668647766, "learning_rate": 7.476727793652011e-05, "loss": 0.004544955212622881, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00456, "step": 199, "tokens/total": 6436016, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 90407} +{"epoch": 0.78125, "grad_norm": 0.7930775880813599, "learning_rate": 7.450608454068415e-05, "loss": 0.017012078315019608, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.01716, "step": 200, "tokens/total": 6468656, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 90849} +{"epoch": 0.78515625, "grad_norm": 0.4090823233127594, "learning_rate": 7.424407941704987e-05, "loss": 0.007651176769286394, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00768, "step": 201, "tokens/total": 6502728, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 91350} +{"epoch": 0.7890625, "grad_norm": 0.7971380352973938, "learning_rate": 7.398127346871986e-05, "loss": 0.04697509482502937, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.0481, "step": 202, "tokens/total": 6535616, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 91805} +{"epoch": 0.79296875, "grad_norm": 0.19888180494308472, "learning_rate": 7.371767763212238e-05, "loss": 0.002735789865255356, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00274, "step": 203, "tokens/total": 6567088, "tokens/train_per_sec_per_gpu": 10.11, "tokens/trainable": 92258} +{"epoch": 0.796875, "grad_norm": 0.3196755647659302, "learning_rate": 7.345330287655617e-05, "loss": 0.0026873000897467136, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00269, "step": 204, "tokens/total": 6600808, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 92721} +{"epoch": 0.80078125, "grad_norm": 1.1855137348175049, "learning_rate": 7.31881602037339e-05, "loss": 0.02920132875442505, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.02963, "step": 205, "tokens/total": 6631384, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 93160} +{"epoch": 0.8046875, "grad_norm": 0.578705370426178, "learning_rate": 7.29222606473245e-05, "loss": 0.01390581764280796, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.014, "step": 206, "tokens/total": 6661888, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 93605} +{"epoch": 0.80859375, "grad_norm": 0.4047410786151886, "learning_rate": 7.265561527249383e-05, "loss": 0.017654532566666603, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.01781, "step": 207, "tokens/total": 6693456, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 94041} +{"epoch": 0.8125, "grad_norm": 0.4396299123764038, "learning_rate": 7.238823517544436e-05, "loss": 0.021741032600402832, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.02198, "step": 208, "tokens/total": 6727840, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 94512} +{"epoch": 0.81640625, "grad_norm": 0.2475583851337433, "learning_rate": 7.212013148295333e-05, "loss": 0.006687437649816275, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00671, "step": 209, "tokens/total": 6761704, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 94994} +{"epoch": 0.8203125, "grad_norm": 0.20696459710597992, "learning_rate": 7.185131535190975e-05, "loss": 0.005950180348008871, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00597, "step": 210, "tokens/total": 6792648, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 95503} +{"epoch": 0.82421875, "grad_norm": 0.3401489555835724, "learning_rate": 7.158179796885005e-05, "loss": 0.00901560764759779, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00906, "step": 211, "tokens/total": 6824424, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 95954} +{"epoch": 0.828125, "grad_norm": 0.26618310809135437, "learning_rate": 7.131159054949273e-05, "loss": 0.009933143854141235, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00998, "step": 212, "tokens/total": 6861216, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 96428} +{"epoch": 0.83203125, "grad_norm": 0.2830636203289032, "learning_rate": 7.104070433827139e-05, "loss": 0.00563441589474678, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00565, "step": 213, "tokens/total": 6892704, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 96828} +{"epoch": 0.8359375, "grad_norm": 0.04769742116332054, "learning_rate": 7.076915060786705e-05, "loss": 0.0010412279516458511, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.00104, "step": 214, "tokens/total": 6924816, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 97270} +{"epoch": 0.83984375, "grad_norm": 0.2630624771118164, "learning_rate": 7.049694065873882e-05, "loss": 0.00765075022354722, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00768, "step": 215, "tokens/total": 6957536, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 97727} +{"epoch": 0.84375, "grad_norm": 0.4201779067516327, "learning_rate": 7.022408581865382e-05, "loss": 0.015786288306117058, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.01591, "step": 216, "tokens/total": 6991448, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 98201} +{"epoch": 0.84765625, "grad_norm": 0.0900149866938591, "learning_rate": 6.99505974422157e-05, "loss": 0.0007865437655709684, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00079, "step": 217, "tokens/total": 7023712, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 98674} +{"epoch": 0.8515625, "grad_norm": 0.12468001246452332, "learning_rate": 6.967648691039213e-05, "loss": 0.001144524896517396, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00115, "step": 218, "tokens/total": 7055040, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 99108} +{"epoch": 0.85546875, "grad_norm": 0.09298628568649292, "learning_rate": 6.940176563004123e-05, "loss": 0.0015823633875697851, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00158, "step": 219, "tokens/total": 7084944, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 99540} +{"epoch": 0.859375, "grad_norm": 0.19187594950199127, "learning_rate": 6.912644503343682e-05, "loss": 0.0007297302363440394, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00073, "step": 220, "tokens/total": 7119264, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 99978} +{"epoch": 0.86328125, "grad_norm": 0.46056097745895386, "learning_rate": 6.885053657779273e-05, "loss": 0.008655089884996414, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.00869, "step": 221, "tokens/total": 7150848, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 100450} +{"epoch": 0.8671875, "grad_norm": 0.0340229831635952, "learning_rate": 6.857405174478604e-05, "loss": 0.00029998531681485474, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.0003, "step": 222, "tokens/total": 7185176, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 100903} +{"epoch": 0.87109375, "grad_norm": 14.3018159866333, "learning_rate": 6.82970020400792e-05, "loss": 0.02595650777220726, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.0263, "step": 223, "tokens/total": 7219024, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 101351} +{"epoch": 0.875, "grad_norm": 0.6315345168113708, "learning_rate": 6.801939899284132e-05, "loss": 0.010323816910386086, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.01038, "step": 224, "tokens/total": 7253816, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 101857} +{"epoch": 0.87890625, "grad_norm": 0.5430828332901001, "learning_rate": 6.774125415526827e-05, "loss": 0.00860072672367096, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00864, "step": 225, "tokens/total": 7286344, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 102328} +{"epoch": 0.8828125, "grad_norm": 0.1795961558818817, "learning_rate": 6.746257910210214e-05, "loss": 0.0018330527236685157, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00183, "step": 226, "tokens/total": 7317952, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 102779} +{"epoch": 0.88671875, "grad_norm": 0.25791528820991516, "learning_rate": 6.718338543014937e-05, "loss": 0.004416844341903925, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00443, "step": 227, "tokens/total": 7347336, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 103228} +{"epoch": 0.890625, "grad_norm": 0.22308826446533203, "learning_rate": 6.69036847577983e-05, "loss": 0.0017280246829614043, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00173, "step": 228, "tokens/total": 7377592, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 103684} +{"epoch": 0.89453125, "grad_norm": 0.25394412875175476, "learning_rate": 6.662348872453553e-05, "loss": 0.003878682618960738, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00389, "step": 229, "tokens/total": 7410184, "tokens/train_per_sec_per_gpu": 10.52, "tokens/trainable": 104145} +{"epoch": 0.8984375, "grad_norm": 0.2938011884689331, "learning_rate": 6.63428089904618e-05, "loss": 0.004207686986774206, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00422, "step": 230, "tokens/total": 7443984, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 104598} +{"epoch": 0.90234375, "grad_norm": 0.14621542394161224, "learning_rate": 6.60616572358065e-05, "loss": 0.0012705157278105617, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00127, "step": 231, "tokens/total": 7472480, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 104998} +{"epoch": 0.90625, "grad_norm": 0.38902539014816284, "learning_rate": 6.578004516044172e-05, "loss": 0.005870713386684656, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00589, "step": 232, "tokens/total": 7505992, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 105465} +{"epoch": 0.91015625, "grad_norm": 0.45703670382499695, "learning_rate": 6.549798448339548e-05, "loss": 0.008526962250471115, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00856, "step": 233, "tokens/total": 7541168, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 105928} +{"epoch": 0.9140625, "grad_norm": 0.10829176008701324, "learning_rate": 6.521548694236384e-05, "loss": 0.0007867110543884337, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00079, "step": 234, "tokens/total": 7574208, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 106365} +{"epoch": 0.91796875, "grad_norm": 0.3705998361110687, "learning_rate": 6.493256429322259e-05, "loss": 0.005956781096756458, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00597, "step": 235, "tokens/total": 7607016, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 106796} +{"epoch": 0.921875, "grad_norm": 0.1788054257631302, "learning_rate": 6.464922830953799e-05, "loss": 0.002768941456452012, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00277, "step": 236, "tokens/total": 7641744, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 107227} +{"epoch": 0.92578125, "grad_norm": 1.3475157022476196, "learning_rate": 6.436549078207688e-05, "loss": 0.018137648701667786, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.0183, "step": 237, "tokens/total": 7673096, "tokens/train_per_sec_per_gpu": 10.16, "tokens/trainable": 107687} +{"epoch": 0.9296875, "grad_norm": 0.08858118206262589, "learning_rate": 6.408136351831592e-05, "loss": 0.0010957409394904971, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.0011, "step": 238, "tokens/total": 7705344, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 108127} +{"epoch": 0.93359375, "grad_norm": 0.009851684793829918, "learning_rate": 6.379685834195036e-05, "loss": 0.00011848208669107407, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00012, "step": 239, "tokens/total": 7740240, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 108633} +{"epoch": 0.9375, "grad_norm": 0.41404205560684204, "learning_rate": 6.351198709240186e-05, "loss": 0.011891478672623634, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.01196, "step": 240, "tokens/total": 7775392, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 109085} +{"epoch": 0.94140625, "grad_norm": 0.1682090312242508, "learning_rate": 6.32267616243259e-05, "loss": 0.0020338150206953287, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00204, "step": 241, "tokens/total": 7807296, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 109548} +{"epoch": 0.9453125, "grad_norm": 0.0638488233089447, "learning_rate": 6.294119380711849e-05, "loss": 0.0005769429262727499, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00058, "step": 242, "tokens/total": 7838200, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 109975} +{"epoch": 0.94921875, "grad_norm": 0.46147263050079346, "learning_rate": 6.265529552442209e-05, "loss": 0.020439472049474716, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.02065, "step": 243, "tokens/total": 7870936, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 110436} +{"epoch": 0.953125, "grad_norm": 0.1019979938864708, "learning_rate": 6.236907867363127e-05, "loss": 0.0014360630884766579, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00144, "step": 244, "tokens/total": 7902512, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 110882} +{"epoch": 0.95703125, "grad_norm": 0.29016947746276855, "learning_rate": 6.208255516539749e-05, "loss": 0.00790666788816452, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00794, "step": 245, "tokens/total": 7933224, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 111355} +{"epoch": 0.9609375, "grad_norm": 0.040031541138887405, "learning_rate": 6.179573692313344e-05, "loss": 0.0007055830210447311, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00071, "step": 246, "tokens/total": 7968456, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 111803} +{"epoch": 0.96484375, "grad_norm": 1.2754937410354614, "learning_rate": 6.150863588251694e-05, "loss": 0.006896344479173422, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00692, "step": 247, "tokens/total": 8003592, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 112270} +{"epoch": 0.96875, "grad_norm": 0.4930363595485687, "learning_rate": 6.122126399099419e-05, "loss": 0.004963584244251251, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00498, "step": 248, "tokens/total": 8040048, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 112778} +{"epoch": 0.97265625, "grad_norm": 0.8609141111373901, "learning_rate": 6.0933633207282615e-05, "loss": 0.023814111948013306, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.0241, "step": 249, "tokens/total": 8074000, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 113284} +{"epoch": 0.9765625, "grad_norm": 0.2539865970611572, "learning_rate": 6.064575550087316e-05, "loss": 0.0032459262292832136, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00325, "step": 250, "tokens/total": 8108304, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 113722} +{"epoch": 0.98046875, "grad_norm": 0.34598883986473083, "learning_rate": 6.0357642851532245e-05, "loss": 0.006299001630395651, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00632, "step": 251, "tokens/total": 8143464, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 114213} +{"epoch": 0.984375, "grad_norm": 0.06467875838279724, "learning_rate": 6.0069307248803294e-05, "loss": 0.0016724197193980217, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00167, "step": 252, "tokens/total": 8175576, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 114634} +{"epoch": 0.98828125, "grad_norm": 0.09702245891094208, "learning_rate": 5.9780760691507635e-05, "loss": 0.002508827019482851, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00251, "step": 253, "tokens/total": 8205528, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 115066} +{"epoch": 0.9921875, "grad_norm": 0.2162650227546692, "learning_rate": 5.9492015187245334e-05, "loss": 0.0023232176899909973, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00233, "step": 254, "tokens/total": 8239424, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 115520} +{"epoch": 0.99609375, "grad_norm": 0.27587637305259705, "learning_rate": 5.920308275189541e-05, "loss": 0.006760668009519577, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00678, "step": 255, "tokens/total": 8270392, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 115921} +{"epoch": 1.0, "grad_norm": 0.6660588979721069, "learning_rate": 5.8913975409115874e-05, "loss": 0.002308790571987629, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00231, "step": 256, "tokens/total": 8302096, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 116384} +{"epoch": 1.00390625, "grad_norm": 0.04630693793296814, "learning_rate": 5.8624705189843395e-05, "loss": 0.0007463301299139857, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00075, "step": 257, "tokens/total": 8337064, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 116871} +{"epoch": 1.0078125, "grad_norm": 0.09998749196529388, "learning_rate": 5.833528413179249e-05, "loss": 0.001369468867778778, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00137, "step": 258, "tokens/total": 8372840, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 117369} +{"epoch": 1.01171875, "grad_norm": 0.21403662860393524, "learning_rate": 5.80457242789548e-05, "loss": 0.002729737898334861, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00273, "step": 259, "tokens/total": 8409728, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 117866} +{"epoch": 1.015625, "grad_norm": 0.2543415427207947, "learning_rate": 5.77560376810977e-05, "loss": 0.004570052493363619, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.00458, "step": 260, "tokens/total": 8442296, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 118330} +{"epoch": 1.01953125, "grad_norm": 0.10628105700016022, "learning_rate": 5.7466236393263005e-05, "loss": 0.0011866381391882896, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00119, "step": 261, "tokens/total": 8474568, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 118776} +{"epoch": 1.0234375, "grad_norm": 0.05050063133239746, "learning_rate": 5.717633247526522e-05, "loss": 0.0006262233946472406, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00063, "step": 262, "tokens/total": 8505520, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 119211} +{"epoch": 1.02734375, "grad_norm": 0.2774001955986023, "learning_rate": 5.688633799118971e-05, "loss": 0.0025599251966923475, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00256, "step": 263, "tokens/total": 8535064, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 119616} +{"epoch": 1.03125, "grad_norm": 0.3486890494823456, "learning_rate": 5.659626500889066e-05, "loss": 0.0013766714837402105, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00138, "step": 264, "tokens/total": 8563600, "tokens/train_per_sec_per_gpu": 7.04, "tokens/trainable": 120086} +{"epoch": 1.03515625, "grad_norm": 0.06636308133602142, "learning_rate": 5.6306125599488905e-05, "loss": 0.0002822543028742075, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00028, "step": 265, "tokens/total": 8592648, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 120525} +{"epoch": 1.0390625, "grad_norm": 0.20859166979789734, "learning_rate": 5.601593183686955e-05, "loss": 0.0015583208296447992, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00156, "step": 266, "tokens/total": 8627328, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 121001} +{"epoch": 1.04296875, "grad_norm": 0.6394239664077759, "learning_rate": 5.572569579717961e-05, "loss": 0.009721489623188972, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00977, "step": 267, "tokens/total": 8659760, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 121465} +{"epoch": 1.046875, "grad_norm": 0.006360857747495174, "learning_rate": 5.543542955832538e-05, "loss": 2.697675517993048e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00003, "step": 268, "tokens/total": 8696968, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 121908} +{"epoch": 1.05078125, "grad_norm": 0.08893278986215591, "learning_rate": 5.514514519946986e-05, "loss": 0.0007558545330539346, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00076, "step": 269, "tokens/total": 8735392, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 122406} +{"epoch": 1.0546875, "grad_norm": 0.007526597008109093, "learning_rate": 5.485485480053015e-05, "loss": 6.897618732182309e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00007, "step": 270, "tokens/total": 8769840, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 122812} +{"epoch": 1.05859375, "grad_norm": 0.16201665997505188, "learning_rate": 5.4564570441674645e-05, "loss": 0.0016182789113372564, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00162, "step": 271, "tokens/total": 8800648, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 123252} +{"epoch": 1.0625, "grad_norm": 0.004136078990995884, "learning_rate": 5.42743042028204e-05, "loss": 5.318524199537933e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00005, "step": 272, "tokens/total": 8832480, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 123692} +{"epoch": 1.06640625, "grad_norm": 0.5230026245117188, "learning_rate": 5.3984068163130464e-05, "loss": 0.010718362405896187, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.01078, "step": 273, "tokens/total": 8864840, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 124162} +{"epoch": 1.0703125, "grad_norm": 0.19641052186489105, "learning_rate": 5.369387440051111e-05, "loss": 0.001509936759248376, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00151, "step": 274, "tokens/total": 8897272, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 124625} +{"epoch": 1.07421875, "grad_norm": 0.010356850922107697, "learning_rate": 5.340373499110935e-05, "loss": 7.021950295893475e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00007, "step": 275, "tokens/total": 8930688, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 125119} +{"epoch": 1.078125, "grad_norm": 0.02857906185090542, "learning_rate": 5.3113662008810304e-05, "loss": 2.894535646191798e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00003, "step": 276, "tokens/total": 8967216, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 125601} +{"epoch": 1.08203125, "grad_norm": 0.12185167521238327, "learning_rate": 5.282366752473479e-05, "loss": 0.00034519704058766365, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00035, "step": 277, "tokens/total": 9001368, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 126075} +{"epoch": 1.0859375, "grad_norm": 0.07032110542058945, "learning_rate": 5.2533763606737005e-05, "loss": 0.00018301111413165927, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00018, "step": 278, "tokens/total": 9036808, "tokens/train_per_sec_per_gpu": 6.91, "tokens/trainable": 126491} +{"epoch": 1.08984375, "grad_norm": 0.048159159719944, "learning_rate": 5.224396231890232e-05, "loss": 0.0003126721130684018, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00031, "step": 279, "tokens/total": 9067840, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 126940} +{"epoch": 1.09375, "grad_norm": 0.031110616400837898, "learning_rate": 5.195427572104522e-05, "loss": 0.00022054325381759554, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00022, "step": 280, "tokens/total": 9098936, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 127376} +{"epoch": 1.09765625, "grad_norm": 0.12877453863620758, "learning_rate": 5.166471586820751e-05, "loss": 0.0011859347578138113, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00119, "step": 281, "tokens/total": 9133264, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 127841} +{"epoch": 1.1015625, "grad_norm": 0.3499983847141266, "learning_rate": 5.1375294810156615e-05, "loss": 0.020554324612021446, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.02077, "step": 282, "tokens/total": 9165008, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 128288} +{"epoch": 1.10546875, "grad_norm": 0.14064759016036987, "learning_rate": 5.1086024590884144e-05, "loss": 0.0010561193339526653, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00106, "step": 283, "tokens/total": 9197952, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 128737} +{"epoch": 1.109375, "grad_norm": 0.10309085249900818, "learning_rate": 5.079691724810461e-05, "loss": 0.0008469627355225384, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00085, "step": 284, "tokens/total": 9232576, "tokens/train_per_sec_per_gpu": 7.11, "tokens/trainable": 129172} +{"epoch": 1.11328125, "grad_norm": 0.05602679401636124, "learning_rate": 5.0507984812754684e-05, "loss": 0.0002952288486994803, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.0003, "step": 285, "tokens/total": 9264568, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 129669} +{"epoch": 1.1171875, "grad_norm": 0.18132242560386658, "learning_rate": 5.021923930849237e-05, "loss": 0.0007499667699448764, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00075, "step": 286, "tokens/total": 9296544, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 130155} +{"epoch": 1.12109375, "grad_norm": 0.002265752060338855, "learning_rate": 4.99306927511967e-05, "loss": 3.195888712070882e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00003, "step": 287, "tokens/total": 9327936, "tokens/train_per_sec_per_gpu": 10.44, "tokens/trainable": 130595} +{"epoch": 1.125, "grad_norm": 0.29320040345191956, "learning_rate": 4.964235714846775e-05, "loss": 0.0033376109786331654, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00334, "step": 288, "tokens/total": 9357984, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 131015} +{"epoch": 1.12890625, "grad_norm": 0.00971532054245472, "learning_rate": 4.9354244499126866e-05, "loss": 8.113842341117561e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00008, "step": 289, "tokens/total": 9388664, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 131481} +{"epoch": 1.1328125, "grad_norm": 0.01348015945404768, "learning_rate": 4.90663667927174e-05, "loss": 9.203009540215135e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00009, "step": 290, "tokens/total": 9420504, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 131930} +{"epoch": 1.13671875, "grad_norm": 0.10506481677293777, "learning_rate": 4.877873600900581e-05, "loss": 0.00036097975680604577, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00036, "step": 291, "tokens/total": 9452664, "tokens/train_per_sec_per_gpu": 6.23, "tokens/trainable": 132388} +{"epoch": 1.140625, "grad_norm": 0.3056443929672241, "learning_rate": 4.849136411748306e-05, "loss": 0.001518878503702581, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00152, "step": 292, "tokens/total": 9488896, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 132860} +{"epoch": 1.14453125, "grad_norm": 0.025334037840366364, "learning_rate": 4.8204263076866574e-05, "loss": 0.00019742122094612569, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.0002, "step": 293, "tokens/total": 9522112, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 133360} +{"epoch": 1.1484375, "grad_norm": 1.993168830871582, "learning_rate": 4.791744483460251e-05, "loss": 0.030282648280262947, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.03075, "step": 294, "tokens/total": 9555632, "tokens/train_per_sec_per_gpu": 6.6, "tokens/trainable": 133793} +{"epoch": 1.15234375, "grad_norm": 0.004125860519707203, "learning_rate": 4.7630921326368736e-05, "loss": 6.766286969650537e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00007, "step": 295, "tokens/total": 9590904, "tokens/train_per_sec_per_gpu": 6.42, "tokens/trainable": 134218} +{"epoch": 1.15625, "grad_norm": 0.5243661999702454, "learning_rate": 4.7344704475577916e-05, "loss": 0.0029606204479932785, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00297, "step": 296, "tokens/total": 9620288, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 134654} +{"epoch": 1.16015625, "grad_norm": 0.6102409958839417, "learning_rate": 4.705880619288153e-05, "loss": 0.018228061497211456, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.0184, "step": 297, "tokens/total": 9651984, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 135056} +{"epoch": 1.1640625, "grad_norm": 1.0299890041351318, "learning_rate": 4.677323837567412e-05, "loss": 0.003996518440544605, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.004, "step": 298, "tokens/total": 9681368, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 135483} +{"epoch": 1.16796875, "grad_norm": 0.03577566519379616, "learning_rate": 4.6488012907598146e-05, "loss": 0.0002531924983486533, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00025, "step": 299, "tokens/total": 9712872, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 135957} +{"epoch": 1.171875, "grad_norm": 1.634060263633728, "learning_rate": 4.620314165804964e-05, "loss": 0.0016193008050322533, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00162, "step": 300, "tokens/total": 9745184, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 136430} +{"epoch": 1.17578125, "grad_norm": 0.12229589372873306, "learning_rate": 4.591863648168407e-05, "loss": 0.0010129810543730855, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00101, "step": 301, "tokens/total": 9778928, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 136896} +{"epoch": 1.1796875, "grad_norm": 0.0698462724685669, "learning_rate": 4.5634509217923135e-05, "loss": 0.0004861065244767815, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00049, "step": 302, "tokens/total": 9810752, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 137331} +{"epoch": 1.18359375, "grad_norm": 0.14298896491527557, "learning_rate": 4.535077169046201e-05, "loss": 0.0011031038593500853, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.0011, "step": 303, "tokens/total": 9843672, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 137768} +{"epoch": 1.1875, "grad_norm": 0.646557092666626, "learning_rate": 4.506743570677743e-05, "loss": 0.008957352489233017, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.009, "step": 304, "tokens/total": 9873752, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 138200} +{"epoch": 1.19140625, "grad_norm": 0.24170273542404175, "learning_rate": 4.478451305763618e-05, "loss": 0.0035356292501091957, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00354, "step": 305, "tokens/total": 9906272, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 138660} +{"epoch": 1.1953125, "grad_norm": 1.036319375038147, "learning_rate": 4.450201551660454e-05, "loss": 0.016283852979540825, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.01642, "step": 306, "tokens/total": 9936296, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 139092} +{"epoch": 1.19921875, "grad_norm": 0.4664401113986969, "learning_rate": 4.4219954839558276e-05, "loss": 0.005832680035382509, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00585, "step": 307, "tokens/total": 9967720, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 139533} +{"epoch": 1.203125, "grad_norm": 0.07596354931592941, "learning_rate": 4.393834276419352e-05, "loss": 0.0005649289232678711, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.00057, "step": 308, "tokens/total": 9997176, "tokens/train_per_sec_per_gpu": 10.81, "tokens/trainable": 140020} +{"epoch": 1.20703125, "grad_norm": 0.38750603795051575, "learning_rate": 4.36571910095382e-05, "loss": 0.009331333450973034, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00938, "step": 309, "tokens/total": 10031872, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 140465} +{"epoch": 1.2109375, "grad_norm": 0.04096437245607376, "learning_rate": 4.337651127546448e-05, "loss": 0.0005047102458775043, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0005, "step": 310, "tokens/total": 10065240, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 140915} +{"epoch": 1.21484375, "grad_norm": 0.02962588518857956, "learning_rate": 4.3096315242201736e-05, "loss": 0.00046920753084123135, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00047, "step": 311, "tokens/total": 10096640, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 141350} +{"epoch": 1.21875, "grad_norm": 0.57095867395401, "learning_rate": 4.2816614569850635e-05, "loss": 0.002149022649973631, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00215, "step": 312, "tokens/total": 10130192, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 141789} +{"epoch": 1.22265625, "grad_norm": 1.0511635541915894, "learning_rate": 4.2537420897897864e-05, "loss": 0.002817284781485796, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00282, "step": 313, "tokens/total": 10164248, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 142266} +{"epoch": 1.2265625, "grad_norm": 1.030124306678772, "learning_rate": 4.225874584473174e-05, "loss": 0.009394904598593712, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00944, "step": 314, "tokens/total": 10194664, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 142704} +{"epoch": 1.23046875, "grad_norm": 0.03157924488186836, "learning_rate": 4.19806010071587e-05, "loss": 0.0004082625382579863, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00041, "step": 315, "tokens/total": 10227768, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 143168} +{"epoch": 1.234375, "grad_norm": 0.3608309328556061, "learning_rate": 4.170299795992081e-05, "loss": 0.005196165759116411, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00521, "step": 316, "tokens/total": 10260464, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 143615} +{"epoch": 1.23828125, "grad_norm": 0.7789857983589172, "learning_rate": 4.142594825521398e-05, "loss": 0.0035978779196739197, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.0036, "step": 317, "tokens/total": 10297528, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 144096} +{"epoch": 1.2421875, "grad_norm": 0.1807830035686493, "learning_rate": 4.114946342220728e-05, "loss": 0.0008980577113106847, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.0009, "step": 318, "tokens/total": 10331448, "tokens/train_per_sec_per_gpu": 7.0, "tokens/trainable": 144543} +{"epoch": 1.24609375, "grad_norm": 0.0230998657643795, "learning_rate": 4.087355496656321e-05, "loss": 0.00046916696010157466, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00047, "step": 319, "tokens/total": 10363152, "tokens/train_per_sec_per_gpu": 10.18, "tokens/trainable": 145016} +{"epoch": 1.25, "grad_norm": 0.3331398665904999, "learning_rate": 4.05982343699588e-05, "loss": 0.0039021919947117567, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00391, "step": 320, "tokens/total": 10397832, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 145511} +{"epoch": 1.25390625, "grad_norm": 0.11015600711107254, "learning_rate": 4.0323513089607876e-05, "loss": 0.0007839949685148895, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00078, "step": 321, "tokens/total": 10431416, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 145978} +{"epoch": 1.2578125, "grad_norm": 1.0501331090927124, "learning_rate": 4.004940255778431e-05, "loss": 0.0091093135997653, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00915, "step": 322, "tokens/total": 10467384, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 146422} +{"epoch": 1.26171875, "grad_norm": 0.053848084062337875, "learning_rate": 3.977591418134619e-05, "loss": 0.000990233733318746, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00099, "step": 323, "tokens/total": 10500792, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 146863} +{"epoch": 1.265625, "grad_norm": 0.1269630491733551, "learning_rate": 3.95030593412612e-05, "loss": 0.0026678028516471386, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00267, "step": 324, "tokens/total": 10532592, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 147286} +{"epoch": 1.26953125, "grad_norm": 0.04474765434861183, "learning_rate": 3.923084939213296e-05, "loss": 0.0006897040875628591, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00069, "step": 325, "tokens/total": 10562392, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 147708} +{"epoch": 1.2734375, "grad_norm": 0.2572008967399597, "learning_rate": 3.895929566172861e-05, "loss": 0.00269767502322793, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.0027, "step": 326, "tokens/total": 10595416, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 148172} +{"epoch": 1.27734375, "grad_norm": 0.0214969702064991, "learning_rate": 3.868840945050728e-05, "loss": 0.0002475652436260134, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00025, "step": 327, "tokens/total": 10630504, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 148604} +{"epoch": 1.28125, "grad_norm": 0.020762672647833824, "learning_rate": 3.841820203114995e-05, "loss": 0.00022814575640950352, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00023, "step": 328, "tokens/total": 10661632, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 149051} +{"epoch": 1.28515625, "grad_norm": 0.12748470902442932, "learning_rate": 3.814868464809027e-05, "loss": 0.0013064831728115678, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00131, "step": 329, "tokens/total": 10691544, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 149453} +{"epoch": 1.2890625, "grad_norm": 0.01952018216252327, "learning_rate": 3.787986851704667e-05, "loss": 0.00031497341115027666, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00032, "step": 330, "tokens/total": 10724768, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 149883} +{"epoch": 1.29296875, "grad_norm": 0.00821278803050518, "learning_rate": 3.7611764824555654e-05, "loss": 0.00012160601909272373, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00012, "step": 331, "tokens/total": 10756008, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 150357} +{"epoch": 1.296875, "grad_norm": 0.021083978936076164, "learning_rate": 3.734438472750619e-05, "loss": 0.00014832285523880273, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00015, "step": 332, "tokens/total": 10793152, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 150828} +{"epoch": 1.30078125, "grad_norm": 0.0074017783626914024, "learning_rate": 3.707773935267552e-05, "loss": 0.00011753005674108863, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00012, "step": 333, "tokens/total": 10827624, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 151314} +{"epoch": 1.3046875, "grad_norm": 0.004245143849402666, "learning_rate": 3.68118397962661e-05, "loss": 5.6669741752557456e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00006, "step": 334, "tokens/total": 10859712, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 151722} +{"epoch": 1.30859375, "grad_norm": 0.017578504979610443, "learning_rate": 3.654669712344384e-05, "loss": 0.00023361635976471007, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00023, "step": 335, "tokens/total": 10888792, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 152178} +{"epoch": 1.3125, "grad_norm": 0.05184459313750267, "learning_rate": 3.628232236787763e-05, "loss": 0.0005210191593505442, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00052, "step": 336, "tokens/total": 10922968, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 152617} +{"epoch": 1.31640625, "grad_norm": 0.028045587241649628, "learning_rate": 3.6018726531280144e-05, "loss": 0.0001862614299170673, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00019, "step": 337, "tokens/total": 10958000, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 153108} +{"epoch": 1.3203125, "grad_norm": 0.31177863478660583, "learning_rate": 3.575592058295017e-05, "loss": 0.0027756865601986647, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00278, "step": 338, "tokens/total": 10989872, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 153589} +{"epoch": 1.32421875, "grad_norm": 0.0877041444182396, "learning_rate": 3.549391545931585e-05, "loss": 0.0005153788952156901, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00052, "step": 339, "tokens/total": 11020784, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 154049} +{"epoch": 1.328125, "grad_norm": 0.028629647567868233, "learning_rate": 3.5232722063479914e-05, "loss": 0.00014160833961796016, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00014, "step": 340, "tokens/total": 11053240, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 154498} +{"epoch": 1.33203125, "grad_norm": 0.004849365912377834, "learning_rate": 3.49723512647657e-05, "loss": 6.569163815584034e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00007, "step": 341, "tokens/total": 11089040, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 154998} +{"epoch": 1.3359375, "grad_norm": 0.4988693296909332, "learning_rate": 3.471281389826491e-05, "loss": 0.010936960577964783, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.011, "step": 342, "tokens/total": 11119672, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 155403} +{"epoch": 1.33984375, "grad_norm": 0.00570918433368206, "learning_rate": 3.4454120764386764e-05, "loss": 8.489342872053385e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00008, "step": 343, "tokens/total": 11155040, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 155858} +{"epoch": 1.34375, "grad_norm": 0.02753206342458725, "learning_rate": 3.4196282628408526e-05, "loss": 0.00013986654812470078, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00014, "step": 344, "tokens/total": 11185952, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 156303} +{"epoch": 1.34765625, "grad_norm": 0.2525104284286499, "learning_rate": 3.3939310220027456e-05, "loss": 0.001274732407182455, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00128, "step": 345, "tokens/total": 11219272, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 156798} +{"epoch": 1.3515625, "grad_norm": 0.006563656497746706, "learning_rate": 3.3683214232914404e-05, "loss": 7.386985089397058e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00007, "step": 346, "tokens/total": 11253424, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 157271} +{"epoch": 1.35546875, "grad_norm": 0.06984228640794754, "learning_rate": 3.342800532426873e-05, "loss": 0.00042997271521016955, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00043, "step": 347, "tokens/total": 11286216, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 157737} +{"epoch": 1.359375, "grad_norm": 0.7181933522224426, "learning_rate": 3.317369411437484e-05, "loss": 0.00409164372831583, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.0041, "step": 348, "tokens/total": 11319480, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 158205} +{"epoch": 1.36328125, "grad_norm": 0.17852352559566498, "learning_rate": 3.292029118616024e-05, "loss": 0.0009943590266630054, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00099, "step": 349, "tokens/total": 11350472, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 158628} +{"epoch": 1.3671875, "grad_norm": 0.22789213061332703, "learning_rate": 3.266780708475511e-05, "loss": 0.0010222418932244182, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00102, "step": 350, "tokens/total": 11383088, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 159106} +{"epoch": 1.37109375, "grad_norm": 0.007777087856084108, "learning_rate": 3.241625231705354e-05, "loss": 6.977075099712238e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00007, "step": 351, "tokens/total": 11418392, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 159565} +{"epoch": 1.375, "grad_norm": 0.07066743820905685, "learning_rate": 3.216563735127618e-05, "loss": 0.000453459593700245, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00045, "step": 352, "tokens/total": 11449784, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 160042} +{"epoch": 1.37890625, "grad_norm": 0.49597421288490295, "learning_rate": 3.191597261653475e-05, "loss": 0.0061216289177536964, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00614, "step": 353, "tokens/total": 11484240, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 160502} +{"epoch": 1.3828125, "grad_norm": 0.27483946084976196, "learning_rate": 3.166726850239794e-05, "loss": 0.003198716789484024, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.0032, "step": 354, "tokens/total": 11513360, "tokens/train_per_sec_per_gpu": 11.02, "tokens/trainable": 160950} +{"epoch": 1.38671875, "grad_norm": 0.0048653134144842625, "learning_rate": 3.141953535845912e-05, "loss": 4.421331323101185e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00004, "step": 355, "tokens/total": 11543112, "tokens/train_per_sec_per_gpu": 10.29, "tokens/trainable": 161400} +{"epoch": 1.390625, "grad_norm": 0.19902509450912476, "learning_rate": 3.11727834939056e-05, "loss": 0.001411551609635353, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00141, "step": 356, "tokens/total": 11573464, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 161863} +{"epoch": 1.39453125, "grad_norm": 0.22877542674541473, "learning_rate": 3.092702317708967e-05, "loss": 0.004259689711034298, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00427, "step": 357, "tokens/total": 11604896, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 162319} +{"epoch": 1.3984375, "grad_norm": 0.04274045675992966, "learning_rate": 3.0682264635101276e-05, "loss": 0.0003141456109005958, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00031, "step": 358, "tokens/total": 11636368, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 162771} +{"epoch": 1.40234375, "grad_norm": 0.528347373008728, "learning_rate": 3.0438518053342407e-05, "loss": 0.002949608489871025, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00295, "step": 359, "tokens/total": 11670216, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 163243} +{"epoch": 1.40625, "grad_norm": 0.2982422709465027, "learning_rate": 3.0195793575103266e-05, "loss": 0.003910040017217398, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00392, "step": 360, "tokens/total": 11707816, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 163707} +{"epoch": 1.41015625, "grad_norm": 0.13354872167110443, "learning_rate": 2.9954101301140146e-05, "loss": 0.0009539583697915077, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00095, "step": 361, "tokens/total": 11742584, "tokens/train_per_sec_per_gpu": 6.92, "tokens/trainable": 164157} +{"epoch": 1.4140625, "grad_norm": 0.026516353711485863, "learning_rate": 2.9713451289255123e-05, "loss": 0.00015238513879012316, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00015, "step": 362, "tokens/total": 11770384, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 164598} +{"epoch": 1.41796875, "grad_norm": 0.008825352415442467, "learning_rate": 2.9473853553877484e-05, "loss": 8.36550461826846e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00008, "step": 363, "tokens/total": 11803208, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 165072} +{"epoch": 1.421875, "grad_norm": 0.03190600499510765, "learning_rate": 2.9235318065647e-05, "loss": 0.0002989383938256651, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.0003, "step": 364, "tokens/total": 11840480, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 165533} +{"epoch": 1.42578125, "grad_norm": 0.14851316809654236, "learning_rate": 2.8997854750998964e-05, "loss": 0.0008614438702352345, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00086, "step": 365, "tokens/total": 11875544, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 165998} +{"epoch": 1.4296875, "grad_norm": 0.0016481396742165089, "learning_rate": 2.8761473491751258e-05, "loss": 3.67386091966182e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00004, "step": 366, "tokens/total": 11907656, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 166433} +{"epoch": 1.43359375, "grad_norm": 0.016533061861991882, "learning_rate": 2.8526184124692883e-05, "loss": 0.00010326763731427491, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.0001, "step": 367, "tokens/total": 11941904, "tokens/train_per_sec_per_gpu": 6.42, "tokens/trainable": 166871} +{"epoch": 1.4375, "grad_norm": 0.035558585077524185, "learning_rate": 2.829199644117484e-05, "loss": 0.0002285260270582512, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00023, "step": 368, "tokens/total": 11973624, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 167281} +{"epoch": 1.44140625, "grad_norm": 0.8639395236968994, "learning_rate": 2.8058920186702553e-05, "loss": 0.004261248745024204, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00427, "step": 369, "tokens/total": 12010360, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 167754} +{"epoch": 1.4453125, "grad_norm": 0.04830053821206093, "learning_rate": 2.782696506053033e-05, "loss": 0.0004549616714939475, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00046, "step": 370, "tokens/total": 12044544, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 168186} +{"epoch": 1.44921875, "grad_norm": 0.6797223687171936, "learning_rate": 2.7596140715257824e-05, "loss": 0.013061068952083588, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.01315, "step": 371, "tokens/total": 12075792, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 168651} +{"epoch": 1.453125, "grad_norm": 0.030942397192120552, "learning_rate": 2.7366456756428184e-05, "loss": 0.0001248713379027322, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00012, "step": 372, "tokens/total": 12108088, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 169108} +{"epoch": 1.45703125, "grad_norm": 0.004251186735928059, "learning_rate": 2.7137922742128486e-05, "loss": 5.8639674534788355e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00006, "step": 373, "tokens/total": 12140888, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 169607} +{"epoch": 1.4609375, "grad_norm": 0.0415261834859848, "learning_rate": 2.691054818259188e-05, "loss": 0.00043496026773937047, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00044, "step": 374, "tokens/total": 12174560, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 170080} +{"epoch": 1.46484375, "grad_norm": 0.16536909341812134, "learning_rate": 2.6684342539801933e-05, "loss": 0.0008985531167127192, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.0009, "step": 375, "tokens/total": 12204848, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 170532} +{"epoch": 1.46875, "grad_norm": 0.2766101658344269, "learning_rate": 2.645931522709877e-05, "loss": 0.0024929018691182137, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.0025, "step": 376, "tokens/total": 12238472, "tokens/train_per_sec_per_gpu": 10.76, "tokens/trainable": 171012} +{"epoch": 1.47265625, "grad_norm": 0.3560233414173126, "learning_rate": 2.6235475608787365e-05, "loss": 0.00394574087113142, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00395, "step": 377, "tokens/total": 12271024, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 171473} +{"epoch": 1.4765625, "grad_norm": 0.0817611888051033, "learning_rate": 2.6012832999747916e-05, "loss": 0.0006717658834531903, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00067, "step": 378, "tokens/total": 12301312, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 171920} +{"epoch": 1.48046875, "grad_norm": 2.5437254905700684, "learning_rate": 2.579139666504821e-05, "loss": 0.01875465176999569, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01893, "step": 379, "tokens/total": 12337072, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 172366} +{"epoch": 1.484375, "grad_norm": 0.0007228774484246969, "learning_rate": 2.557117581955798e-05, "loss": 1.4817262126598507e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00001, "step": 380, "tokens/total": 12368000, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 172832} +{"epoch": 1.48828125, "grad_norm": 0.01932358741760254, "learning_rate": 2.5352179627565532e-05, "loss": 0.00019191575120203197, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00019, "step": 381, "tokens/total": 12399880, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 173272} +{"epoch": 1.4921875, "grad_norm": 0.03116130828857422, "learning_rate": 2.5134417202396277e-05, "loss": 0.00014730755356140435, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00015, "step": 382, "tokens/total": 12432288, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 173679} +{"epoch": 1.49609375, "grad_norm": 0.17598289251327515, "learning_rate": 2.491789760603361e-05, "loss": 0.0010134581243619323, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00101, "step": 383, "tokens/total": 12465592, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 174109} +{"epoch": 1.5, "grad_norm": 0.2372361719608307, "learning_rate": 2.4702629848741764e-05, "loss": 0.0018633726285770535, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00187, "step": 384, "tokens/total": 12498200, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 174564} +{"epoch": 1.50390625, "grad_norm": 0.08262418955564499, "learning_rate": 2.4488622888690785e-05, "loss": 0.0008347253897227347, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00084, "step": 385, "tokens/total": 12530280, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 175004} +{"epoch": 1.5078125, "grad_norm": 0.009167054668068886, "learning_rate": 2.427588563158384e-05, "loss": 6.561589543707669e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00007, "step": 386, "tokens/total": 12563360, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 175490} +{"epoch": 1.51171875, "grad_norm": 0.014651814475655556, "learning_rate": 2.406442693028651e-05, "loss": 0.00021699028729926795, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00022, "step": 387, "tokens/total": 12593032, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 175911} +{"epoch": 1.515625, "grad_norm": 0.005606567952781916, "learning_rate": 2.3854255584458547e-05, "loss": 8.166502811945975e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00008, "step": 388, "tokens/total": 12628272, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 176338} +{"epoch": 1.51953125, "grad_norm": 0.03236030414700508, "learning_rate": 2.3645380340187508e-05, "loss": 0.0003401484282221645, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00034, "step": 389, "tokens/total": 12661368, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 176819} +{"epoch": 1.5234375, "grad_norm": 0.05807972699403763, "learning_rate": 2.3437809889624914e-05, "loss": 0.0003991704434156418, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.77, "memory/max_allocated (GiB)": 63.77, "ppl": 1.0004, "step": 390, "tokens/total": 12691168, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 177270} +{"epoch": 1.52734375, "grad_norm": 0.014399913139641285, "learning_rate": 2.3231552870624487e-05, "loss": 0.0001420619955752045, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00014, "step": 391, "tokens/total": 12721264, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 177696} +{"epoch": 1.53125, "grad_norm": 0.10554759949445724, "learning_rate": 2.3026617866382657e-05, "loss": 0.0006661840598098934, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00067, "step": 392, "tokens/total": 12753640, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 178193} +{"epoch": 1.53515625, "grad_norm": 0.012867743149399757, "learning_rate": 2.2823013405081507e-05, "loss": 0.0001275216491194442, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00013, "step": 393, "tokens/total": 12787056, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 178690} +{"epoch": 1.5390625, "grad_norm": 0.03274818882346153, "learning_rate": 2.2620747959533722e-05, "loss": 0.00033195706782862544, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00033, "step": 394, "tokens/total": 12819312, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 179187} +{"epoch": 1.54296875, "grad_norm": 0.07213786989450455, "learning_rate": 2.2419829946830123e-05, "loss": 0.00038671086076647043, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00039, "step": 395, "tokens/total": 12853376, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 179638} +{"epoch": 1.546875, "grad_norm": 0.061176788061857224, "learning_rate": 2.2220267727989325e-05, "loss": 0.00032595093944109976, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00033, "step": 396, "tokens/total": 12882960, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 180091} +{"epoch": 1.55078125, "grad_norm": 0.6336117386817932, "learning_rate": 2.202206960760984e-05, "loss": 0.008552948012948036, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00859, "step": 397, "tokens/total": 12917016, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 180598} +{"epoch": 1.5546875, "grad_norm": 0.003991969395428896, "learning_rate": 2.182524383352446e-05, "loss": 5.3843796194996685e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00005, "step": 398, "tokens/total": 12950056, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 181051} +{"epoch": 1.55859375, "grad_norm": 0.31831613183021545, "learning_rate": 2.1629798596457056e-05, "loss": 0.002389811910688877, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00239, "step": 399, "tokens/total": 12981728, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 181501} +{"epoch": 1.5625, "grad_norm": 0.16491231322288513, "learning_rate": 2.1435742029681725e-05, "loss": 0.0013286549365147948, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00133, "step": 400, "tokens/total": 13011464, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 181918} +{"epoch": 1.56640625, "grad_norm": 0.08228087425231934, "learning_rate": 2.124308220868431e-05, "loss": 0.0006700996891595423, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00067, "step": 401, "tokens/total": 13044432, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 182373} +{"epoch": 1.5703125, "grad_norm": 0.033719152212142944, "learning_rate": 2.105182715082638e-05, "loss": 0.0003547662345226854, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00035, "step": 402, "tokens/total": 13075608, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 182850} +{"epoch": 1.57421875, "grad_norm": 0.44366368651390076, "learning_rate": 2.0861984815011552e-05, "loss": 0.006343469489365816, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00636, "step": 403, "tokens/total": 13107352, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 183281} +{"epoch": 1.578125, "grad_norm": 0.005497108679264784, "learning_rate": 2.0673563101354323e-05, "loss": 9.315853822045028e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00009, "step": 404, "tokens/total": 13140736, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 183711} +{"epoch": 1.58203125, "grad_norm": 0.28750503063201904, "learning_rate": 2.0486569850851317e-05, "loss": 0.002932172967121005, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00294, "step": 405, "tokens/total": 13174040, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 184174} +{"epoch": 1.5859375, "grad_norm": 0.10820809751749039, "learning_rate": 2.0301012845054956e-05, "loss": 0.0014606330078095198, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00146, "step": 406, "tokens/total": 13207232, "tokens/train_per_sec_per_gpu": 6.68, "tokens/trainable": 184589} +{"epoch": 1.58984375, "grad_norm": 0.003268241649493575, "learning_rate": 2.011689980574966e-05, "loss": 5.5214433814398944e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00006, "step": 407, "tokens/total": 13241160, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 185009} +{"epoch": 1.59375, "grad_norm": 0.036557454615831375, "learning_rate": 1.993423839463052e-05, "loss": 0.0003973493876401335, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.0004, "step": 408, "tokens/total": 13273104, "tokens/train_per_sec_per_gpu": 10.44, "tokens/trainable": 185466} +{"epoch": 1.59765625, "grad_norm": 0.005464720074087381, "learning_rate": 1.975303621298445e-05, "loss": 6.516729627037421e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00007, "step": 409, "tokens/total": 13308136, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 185918} +{"epoch": 1.6015625, "grad_norm": 0.4787731170654297, "learning_rate": 1.957330080137385e-05, "loss": 0.008953151293098927, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00899, "step": 410, "tokens/total": 13337816, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 186369} +{"epoch": 1.60546875, "grad_norm": 0.4380701184272766, "learning_rate": 1.9395039639322864e-05, "loss": 0.0026687136851251125, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00267, "step": 411, "tokens/total": 13372576, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 186844} +{"epoch": 1.609375, "grad_norm": 0.011000371538102627, "learning_rate": 1.9218260145006073e-05, "loss": 0.00010051572462543845, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.0001, "step": 412, "tokens/total": 13406664, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 187330} +{"epoch": 1.61328125, "grad_norm": 0.023104000836610794, "learning_rate": 1.904296967493982e-05, "loss": 0.0002464408753439784, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00025, "step": 413, "tokens/total": 13438256, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 187786} +{"epoch": 1.6171875, "grad_norm": 0.07257946580648422, "learning_rate": 1.8869175523676064e-05, "loss": 0.000789700192399323, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00079, "step": 414, "tokens/total": 13472096, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 188281} +{"epoch": 1.62109375, "grad_norm": 0.2137950211763382, "learning_rate": 1.869688492349885e-05, "loss": 0.0031437042634934187, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00315, "step": 415, "tokens/total": 13504192, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 188733} +{"epoch": 1.625, "grad_norm": 0.3046126961708069, "learning_rate": 1.85261050441233e-05, "loss": 0.00316228368319571, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00317, "step": 416, "tokens/total": 13537392, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 189189} +{"epoch": 1.62890625, "grad_norm": 0.2145603597164154, "learning_rate": 1.8356842992397304e-05, "loss": 0.002134476788341999, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00214, "step": 417, "tokens/total": 13569344, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 189629} +{"epoch": 1.6328125, "grad_norm": 0.02358471229672432, "learning_rate": 1.8189105812005714e-05, "loss": 0.0003364396980032325, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00034, "step": 418, "tokens/total": 13602200, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 190039} +{"epoch": 1.63671875, "grad_norm": 0.15057270228862762, "learning_rate": 1.802290048317732e-05, "loss": 0.000724355923011899, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00072, "step": 419, "tokens/total": 13632640, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 190457} +{"epoch": 1.640625, "grad_norm": 0.005438550841063261, "learning_rate": 1.785823392239424e-05, "loss": 5.8179357438348234e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00006, "step": 420, "tokens/total": 13664920, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 190898} +{"epoch": 1.64453125, "grad_norm": 0.287230521440506, "learning_rate": 1.7695112982104225e-05, "loss": 0.009049482643604279, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00909, "step": 421, "tokens/total": 13697144, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 191337} +{"epoch": 1.6484375, "grad_norm": 0.010260015726089478, "learning_rate": 1.7533544450435433e-05, "loss": 8.484473073622212e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00008, "step": 422, "tokens/total": 13724496, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 191770} +{"epoch": 1.65234375, "grad_norm": 0.0006755684735253453, "learning_rate": 1.7373535050913946e-05, "loss": 1.3893074537918437e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00001, "step": 423, "tokens/total": 13757880, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 192207} +{"epoch": 1.65625, "grad_norm": 0.08689194917678833, "learning_rate": 1.721509144218405e-05, "loss": 0.00122298882342875, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00122, "step": 424, "tokens/total": 13787648, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 192637} +{"epoch": 1.66015625, "grad_norm": 0.11493182927370071, "learning_rate": 1.705822021773101e-05, "loss": 0.0013251196360215545, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00133, "step": 425, "tokens/total": 13819720, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 193123} +{"epoch": 1.6640625, "grad_norm": 0.35811546444892883, "learning_rate": 1.69029279056068e-05, "loss": 0.007770197931677103, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.0078, "step": 426, "tokens/total": 13854944, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 193564} +{"epoch": 1.66796875, "grad_norm": 0.02157219499349594, "learning_rate": 1.6749220968158415e-05, "loss": 0.00021519872825592756, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00022, "step": 427, "tokens/total": 13887824, "tokens/train_per_sec_per_gpu": 10.37, "tokens/trainable": 194070} +{"epoch": 1.671875, "grad_norm": 0.05140364542603493, "learning_rate": 1.659710580175893e-05, "loss": 0.0006684979889541864, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00067, "step": 428, "tokens/total": 13918192, "tokens/train_per_sec_per_gpu": 10.13, "tokens/trainable": 194569} +{"epoch": 1.67578125, "grad_norm": 0.36400240659713745, "learning_rate": 1.644658873654133e-05, "loss": 0.004708757158368826, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00472, "step": 429, "tokens/total": 13947440, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 195006} +{"epoch": 1.6796875, "grad_norm": 0.048836879432201385, "learning_rate": 1.629767603613508e-05, "loss": 0.0004909491399303079, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00049, "step": 430, "tokens/total": 13974360, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 195406} +{"epoch": 1.68359375, "grad_norm": 0.41626161336898804, "learning_rate": 1.615037389740547e-05, "loss": 0.0023434325121343136, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00235, "step": 431, "tokens/total": 14006592, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 195850} +{"epoch": 1.6875, "grad_norm": 0.5329418182373047, "learning_rate": 1.600468845019576e-05, "loss": 0.015981530770659447, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.01611, "step": 432, "tokens/total": 14040776, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 196330} +{"epoch": 1.69140625, "grad_norm": 0.0018424594309180975, "learning_rate": 1.5860625757072092e-05, "loss": 2.8777732950402424e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00003, "step": 433, "tokens/total": 14073040, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 196778} +{"epoch": 1.6953125, "grad_norm": 0.13807304203510284, "learning_rate": 1.571819181307116e-05, "loss": 0.0014956474769860506, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.0015, "step": 434, "tokens/total": 14107624, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 197250} +{"epoch": 1.69921875, "grad_norm": 0.026920512318611145, "learning_rate": 1.557739254545075e-05, "loss": 0.0002722979406826198, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00027, "step": 435, "tokens/total": 14137784, "tokens/train_per_sec_per_gpu": 10.64, "tokens/trainable": 197722} +{"epoch": 1.703125, "grad_norm": 0.056674614548683167, "learning_rate": 1.543823381344311e-05, "loss": 0.0009573812130838633, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00096, "step": 436, "tokens/total": 14167776, "tokens/train_per_sec_per_gpu": 10.76, "tokens/trainable": 198196} +{"epoch": 1.70703125, "grad_norm": 0.032456763088703156, "learning_rate": 1.5300721408011114e-05, "loss": 0.00031556226895190775, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00032, "step": 437, "tokens/total": 14199752, "tokens/train_per_sec_per_gpu": 10.25, "tokens/trainable": 198653} +{"epoch": 1.7109375, "grad_norm": 0.1039852648973465, "learning_rate": 1.5164861051607254e-05, "loss": 0.0010388633236289024, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.00104, "step": 438, "tokens/total": 14230344, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 199061} +{"epoch": 1.71484375, "grad_norm": 0.015352973714470863, "learning_rate": 1.5030658397935521e-05, "loss": 0.00010403051419416443, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.0001, "step": 439, "tokens/total": 14263808, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 199530} +{"epoch": 1.71875, "grad_norm": 0.02235415205359459, "learning_rate": 1.4898119031716104e-05, "loss": 0.00026342764613218606, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00026, "step": 440, "tokens/total": 14295864, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 199988} +{"epoch": 1.72265625, "grad_norm": 0.019459327682852745, "learning_rate": 1.476724846845306e-05, "loss": 0.0003354785731062293, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00034, "step": 441, "tokens/total": 14331688, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 200465} +{"epoch": 1.7265625, "grad_norm": 0.027739981189370155, "learning_rate": 1.463805215420471e-05, "loss": 0.0003108362725470215, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00031, "step": 442, "tokens/total": 14364688, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 200935} +{"epoch": 1.73046875, "grad_norm": 0.030542761087417603, "learning_rate": 1.451053546535705e-05, "loss": 0.00021034384553786367, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00021, "step": 443, "tokens/total": 14397832, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 201391} +{"epoch": 1.734375, "grad_norm": 0.004538319539278746, "learning_rate": 1.438470370840001e-05, "loss": 6.191182910697535e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00006, "step": 444, "tokens/total": 14431864, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 201863} +{"epoch": 1.73828125, "grad_norm": 0.06082076579332352, "learning_rate": 1.4260562119706606e-05, "loss": 0.00015763573173899204, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00016, "step": 445, "tokens/total": 14462152, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 202337} +{"epoch": 1.7421875, "grad_norm": 0.10039421170949936, "learning_rate": 1.413811586531508e-05, "loss": 0.00113344332203269, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00113, "step": 446, "tokens/total": 14493072, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 202778} +{"epoch": 1.74609375, "grad_norm": 0.0678224191069603, "learning_rate": 1.4017370040713884e-05, "loss": 0.0007721030269749463, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00077, "step": 447, "tokens/total": 14527392, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 203227} +{"epoch": 1.75, "grad_norm": 0.1228114664554596, "learning_rate": 1.3898329670629645e-05, "loss": 0.0011060857214033604, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00111, "step": 448, "tokens/total": 14561992, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 203672} +{"epoch": 1.75390625, "grad_norm": 0.001098781474865973, "learning_rate": 1.3780999708818058e-05, "loss": 2.2546581021742895e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00002, "step": 449, "tokens/total": 14592056, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 204122} +{"epoch": 1.7578125, "grad_norm": 0.02671482227742672, "learning_rate": 1.3665385037857758e-05, "loss": 0.00030689319828525186, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00031, "step": 450, "tokens/total": 14622816, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 204575} +{"epoch": 1.76171875, "grad_norm": 0.017434151843190193, "learning_rate": 1.3551490468947126e-05, "loss": 0.0003326493897475302, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00033, "step": 451, "tokens/total": 14653272, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 205021} +{"epoch": 1.765625, "grad_norm": 0.10026439279317856, "learning_rate": 1.3439320741704075e-05, "loss": 0.0015905228210613132, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00159, "step": 452, "tokens/total": 14683952, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 205444} +{"epoch": 1.76953125, "grad_norm": 0.15974977612495422, "learning_rate": 1.3328880523968808e-05, "loss": 0.001736202510073781, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00174, "step": 453, "tokens/total": 14716256, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 205907} +{"epoch": 1.7734375, "grad_norm": 0.17345404624938965, "learning_rate": 1.3220174411609587e-05, "loss": 0.002355637727305293, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00236, "step": 454, "tokens/total": 14749088, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 206346} +{"epoch": 1.77734375, "grad_norm": 0.005401021800935268, "learning_rate": 1.3113206928331471e-05, "loss": 6.429815402952954e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00006, "step": 455, "tokens/total": 14777384, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 206808} +{"epoch": 1.78125, "grad_norm": 0.00554168364033103, "learning_rate": 1.300798252548806e-05, "loss": 8.413749310420826e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00008, "step": 456, "tokens/total": 14809640, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 207260} +{"epoch": 1.78515625, "grad_norm": 0.01617647148668766, "learning_rate": 1.2904505581896265e-05, "loss": 0.0001566685241414234, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00016, "step": 457, "tokens/total": 14843728, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 207733} +{"epoch": 1.7890625, "grad_norm": 0.042999353259801865, "learning_rate": 1.2802780403654082e-05, "loss": 0.00033470988273620605, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00033, "step": 458, "tokens/total": 14876256, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 208180} +{"epoch": 1.79296875, "grad_norm": 0.019031906500458717, "learning_rate": 1.2702811223961408e-05, "loss": 0.00021389531320892274, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00021, "step": 459, "tokens/total": 14907000, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 208641} +{"epoch": 1.796875, "grad_norm": 0.20339292287826538, "learning_rate": 1.2604602202943861e-05, "loss": 0.002558755688369274, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00256, "step": 460, "tokens/total": 14937704, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 209084} +{"epoch": 1.80078125, "grad_norm": 0.4149264097213745, "learning_rate": 1.2508157427479686e-05, "loss": 0.005080587230622768, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00509, "step": 461, "tokens/total": 14966920, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 209530} +{"epoch": 1.8046875, "grad_norm": 0.12335183471441269, "learning_rate": 1.2413480911029655e-05, "loss": 0.0020809040870517492, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00208, "step": 462, "tokens/total": 14996040, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 209969} +{"epoch": 1.80859375, "grad_norm": 0.00427718460559845, "learning_rate": 1.2320576593470082e-05, "loss": 5.5784570577088743e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00006, "step": 463, "tokens/total": 15029096, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 210396} +{"epoch": 1.8125, "grad_norm": 0.43393805623054504, "learning_rate": 1.2229448340928828e-05, "loss": 0.00217789551243186, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00218, "step": 464, "tokens/total": 15063696, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 210904} +{"epoch": 1.81640625, "grad_norm": 0.014350879937410355, "learning_rate": 1.2140099945624458e-05, "loss": 0.00012333535414654762, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00012, "step": 465, "tokens/total": 15098264, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 211362} +{"epoch": 1.8203125, "grad_norm": 0.020240193232893944, "learning_rate": 1.205253512570841e-05, "loss": 0.00020556137314997613, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00021, "step": 466, "tokens/total": 15130464, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 211810} +{"epoch": 1.82421875, "grad_norm": 0.04768497869372368, "learning_rate": 1.1966757525110255e-05, "loss": 0.0005749461706727743, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00058, "step": 467, "tokens/total": 15158736, "tokens/train_per_sec_per_gpu": 10.11, "tokens/trainable": 212261} +{"epoch": 1.828125, "grad_norm": 0.31383833289146423, "learning_rate": 1.1882770713386095e-05, "loss": 0.005060291383415461, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00507, "step": 468, "tokens/total": 15193648, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 212736} +{"epoch": 1.83203125, "grad_norm": 0.023296548053622246, "learning_rate": 1.180057818556998e-05, "loss": 0.0002265418879687786, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00023, "step": 469, "tokens/total": 15227240, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 213211} +{"epoch": 1.8359375, "grad_norm": 0.001666472409851849, "learning_rate": 1.1720183362028494e-05, "loss": 3.0583232728531584e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00003, "step": 470, "tokens/total": 15259968, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 213676} +{"epoch": 1.83984375, "grad_norm": 0.11081098020076752, "learning_rate": 1.1641589588318387e-05, "loss": 0.0007121448870748281, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00071, "step": 471, "tokens/total": 15292480, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 214121} +{"epoch": 1.84375, "grad_norm": 0.0007735372637398541, "learning_rate": 1.1564800135047418e-05, "loss": 1.1829449249489699e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00001, "step": 472, "tokens/total": 15324976, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 214609} +{"epoch": 1.84765625, "grad_norm": 0.42374637722969055, "learning_rate": 1.148981819773816e-05, "loss": 0.004419208038598299, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00443, "step": 473, "tokens/total": 15356448, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 215026} +{"epoch": 1.8515625, "grad_norm": 0.008834104053676128, "learning_rate": 1.1416646896695086e-05, "loss": 7.783863111399114e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00008, "step": 474, "tokens/total": 15389192, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 215478} +{"epoch": 1.85546875, "grad_norm": 0.008808311074972153, "learning_rate": 1.1345289276874717e-05, "loss": 8.146795880747959e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00008, "step": 475, "tokens/total": 15422728, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 215949} +{"epoch": 1.859375, "grad_norm": 0.05558885633945465, "learning_rate": 1.1275748307758873e-05, "loss": 0.00046719599049538374, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00047, "step": 476, "tokens/total": 15457104, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 216417} +{"epoch": 1.86328125, "grad_norm": 0.0071791610680520535, "learning_rate": 1.1208026883231147e-05, "loss": 6.520887109218165e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00007, "step": 477, "tokens/total": 15486592, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 216832} +{"epoch": 1.8671875, "grad_norm": 0.20910972356796265, "learning_rate": 1.1142127821456433e-05, "loss": 0.0027622038032859564, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00277, "step": 478, "tokens/total": 15518368, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 217282} +{"epoch": 1.87109375, "grad_norm": 0.030637597665190697, "learning_rate": 1.1078053864763674e-05, "loss": 0.000227990880375728, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00023, "step": 479, "tokens/total": 15551256, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 217734} +{"epoch": 1.875, "grad_norm": 0.22280555963516235, "learning_rate": 1.1015807679531756e-05, "loss": 0.0023400643840432167, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00234, "step": 480, "tokens/total": 15581480, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 218182} +{"epoch": 1.87890625, "grad_norm": 0.0430695042014122, "learning_rate": 1.0955391856078528e-05, "loss": 0.0004741087614092976, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00047, "step": 481, "tokens/total": 15615104, "tokens/train_per_sec_per_gpu": 10.32, "tokens/trainable": 218656} +{"epoch": 1.8828125, "grad_norm": 0.10938724130392075, "learning_rate": 1.0896808908553007e-05, "loss": 0.0008200565353035927, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00082, "step": 482, "tokens/total": 15645384, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 219104} +{"epoch": 1.88671875, "grad_norm": 0.3161788582801819, "learning_rate": 1.0840061274830763e-05, "loss": 0.004756359849125147, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00477, "step": 483, "tokens/total": 15676600, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 219570} +{"epoch": 1.890625, "grad_norm": 0.008533750660717487, "learning_rate": 1.0785151316412473e-05, "loss": 0.00010718106932472438, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00011, "step": 484, "tokens/total": 15709712, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 220019} +{"epoch": 1.89453125, "grad_norm": 0.05633552372455597, "learning_rate": 1.0732081318325639e-05, "loss": 0.0006080578314140439, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00061, "step": 485, "tokens/total": 15747848, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 220517} +{"epoch": 1.8984375, "grad_norm": 0.20420777797698975, "learning_rate": 1.0680853489029501e-05, "loss": 0.0020957449451088905, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.0021, "step": 486, "tokens/total": 15782416, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 220947} +{"epoch": 1.90234375, "grad_norm": 0.05108308047056198, "learning_rate": 1.0631469960323152e-05, "loss": 0.0004632312629837543, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00046, "step": 487, "tokens/total": 15815456, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 221383} +{"epoch": 1.90625, "grad_norm": 0.0029722247272729874, "learning_rate": 1.0583932787256783e-05, "loss": 3.7961031921440735e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00004, "step": 488, "tokens/total": 15846496, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 221803} +{"epoch": 1.91015625, "grad_norm": 0.0034396310802549124, "learning_rate": 1.0538243948046206e-05, "loss": 3.8954160118009895e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00004, "step": 489, "tokens/total": 15878784, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 222249} +{"epoch": 1.9140625, "grad_norm": 0.001023614197038114, "learning_rate": 1.0494405343990523e-05, "loss": 1.6405734641011804e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00002, "step": 490, "tokens/total": 15909616, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 222693} +{"epoch": 1.91796875, "grad_norm": 0.018334051594138145, "learning_rate": 1.0452418799392985e-05, "loss": 0.0001565094426041469, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00016, "step": 491, "tokens/total": 15941400, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 223153} +{"epoch": 1.921875, "grad_norm": 0.0009683431126177311, "learning_rate": 1.0412286061485102e-05, "loss": 1.5904231986496598e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00002, "step": 492, "tokens/total": 15975440, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 223620} +{"epoch": 1.92578125, "grad_norm": 0.003995650913566351, "learning_rate": 1.03740088003539e-05, "loss": 5.473483543028124e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00005, "step": 493, "tokens/total": 16004224, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 224061} +{"epoch": 1.9296875, "grad_norm": 0.006819907575845718, "learning_rate": 1.0337588608872463e-05, "loss": 7.640985859325156e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00008, "step": 494, "tokens/total": 16032976, "tokens/train_per_sec_per_gpu": 10.56, "tokens/trainable": 224520} +{"epoch": 1.93359375, "grad_norm": 0.3412066102027893, "learning_rate": 1.0303027002633622e-05, "loss": 0.01021089218556881, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.01026, "step": 495, "tokens/total": 16065312, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 225018} +{"epoch": 1.9375, "grad_norm": 0.006664399988949299, "learning_rate": 1.0270325419886884e-05, "loss": 4.645694571081549e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00005, "step": 496, "tokens/total": 16097552, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 225472} +{"epoch": 1.94140625, "grad_norm": 0.010663237422704697, "learning_rate": 1.0239485221478599e-05, "loss": 0.00011937427188968286, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00012, "step": 497, "tokens/total": 16132112, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 225935} +{"epoch": 1.9453125, "grad_norm": 0.4693506062030792, "learning_rate": 1.0210507690795292e-05, "loss": 0.0004994724877178669, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.0005, "step": 498, "tokens/total": 16163632, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 226360} +{"epoch": 1.94921875, "grad_norm": 0.32308849692344666, "learning_rate": 1.0183394033710305e-05, "loss": 0.0023277942091226578, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00233, "step": 499, "tokens/total": 16194632, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 226811} +{"epoch": 1.953125, "grad_norm": 0.004355002194643021, "learning_rate": 1.0158145378533583e-05, "loss": 5.364768367144279e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00005, "step": 500, "tokens/total": 16225280, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 227278} +{"epoch": 1.95703125, "grad_norm": 0.008431102149188519, "learning_rate": 1.0134762775964726e-05, "loss": 8.582424197811633e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00009, "step": 501, "tokens/total": 16257400, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 227768} +{"epoch": 1.9609375, "grad_norm": 0.0009504699846729636, "learning_rate": 1.0113247199049278e-05, "loss": 1.6218771634157747e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00002, "step": 502, "tokens/total": 16290472, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 228249} +{"epoch": 1.96484375, "grad_norm": 0.13494570553302765, "learning_rate": 1.0093599543138205e-05, "loss": 0.0016923316288739443, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00169, "step": 503, "tokens/total": 16324232, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 228726} +{"epoch": 1.96875, "grad_norm": 0.12443181872367859, "learning_rate": 1.0075820625850675e-05, "loss": 0.0008010470191948116, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.0008, "step": 504, "tokens/total": 16360464, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 229177} +{"epoch": 1.97265625, "grad_norm": 0.003922738134860992, "learning_rate": 1.0059911187040013e-05, "loss": 4.100315345567651e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00004, "step": 505, "tokens/total": 16393136, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 229609} +{"epoch": 1.9765625, "grad_norm": 0.01966475136578083, "learning_rate": 1.0045871888762893e-05, "loss": 0.00021107659267727286, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00021, "step": 506, "tokens/total": 16428488, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 230078} +{"epoch": 1.98046875, "grad_norm": 0.0026661790907382965, "learning_rate": 1.003370331525184e-05, "loss": 4.8168472858378664e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00005, "step": 507, "tokens/total": 16462616, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 230549} +{"epoch": 1.984375, "grad_norm": 0.002807807410135865, "learning_rate": 1.002340597289085e-05, "loss": 2.2056156012695283e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00002, "step": 508, "tokens/total": 16495600, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 230999} +{"epoch": 1.98828125, "grad_norm": 0.020583534613251686, "learning_rate": 1.0014980290194387e-05, "loss": 0.00018507646746002138, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00019, "step": 509, "tokens/total": 16525504, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 231439} +{"epoch": 1.9921875, "grad_norm": 0.22010613977909088, "learning_rate": 1.0008426617789489e-05, "loss": 0.0012660352513194084, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00127, "step": 510, "tokens/total": 16556960, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 231901} +{"epoch": 1.99609375, "grad_norm": 0.021346893161535263, "learning_rate": 1.0003745228401215e-05, "loss": 0.00023380901257041842, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00023, "step": 511, "tokens/total": 16588024, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 232362} +{"epoch": 2.0, "grad_norm": 0.02954045869410038, "learning_rate": 1.0000936316841296e-05, "loss": 0.00012073743710061535, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00012, "step": 512, "tokens/total": 16622040, "tokens/train_per_sec_per_gpu": 6.49, "tokens/trainable": 232768} diff --git a/gemma3_27b_50m/control/training/dolci/axolotl.yaml b/gemma3_27b_50m/control/training/dolci/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78a7a61d5f25b6514b915297b1c032c544d37d6b --- /dev/null +++ b/gemma3_27b_50m/control/training/dolci/axolotl.yaml @@ -0,0 +1,57 @@ +base_model: /workspace/final_v1/gemma3_27b_50m/control/midtrain/checkpoints/checkpoint-381 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_50m/control/data/dolci.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: jinja +chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja +dataset_prepared_path: /workspace/final_v1/gemma3_27b_50m/control/dolci/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 2 +gradient_accumulation_steps: 16 +num_epochs: 1 +max_steps: 48 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_steps: 10 +train_on_inputs: false +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 2 +checkpoint_schedule: +- 43 +- 48 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_50m/control/dolci/checkpoints diff --git a/gemma3_27b_50m/control/training/dolci/train.log b/gemma3_27b_50m/control/training/dolci/train.log new file mode 100644 index 0000000000000000000000000000000000000000..e7ab62755fa73092a38af60769f99a2dd62aa91b --- /dev/null +++ b/gemma3_27b_50m/control/training/dolci/train.log @@ -0,0 +1,482 @@ +[2026-09-02 06:08:03,597] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 06:08:05.501000 279094 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:08:05.521000 279094 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `8` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 06:08:15,311] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 06:08:15,329] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 06:08:15,332] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 06:08:15,345] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 06:08:15,353] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 06:08:15,357] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 06:08:15,375] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 06:08:15,388] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 06:08:19.609000 279498 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:08:19.628000 279498 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:08:20.089000 279496 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:08:20.107000 279496 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:08:20.408000 279497 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:08:20.426000 279497 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:08:20.492000 279495 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:08:20.514000 279495 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:08:20.546000 279494 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:08:20.564000 279494 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:08:20.607000 279493 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:08:20.625000 279493 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:08:20.677000 279492 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:08:20.687000 279499 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:08:20.696000 279492 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:08:20.706000 279499 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 06:08:23,453] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 06:08:23,455] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 06:08:23,455] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 06:08:23,477] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 06:08:23,521] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 06:08:23,521] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-02 06:08:23,521] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-02 06:08:23,521] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-02 06:08:23,880] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_50m/control/dolci/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_50m/control/midtrain/checkpoints/checkpoint-381", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 256, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 8, + "n_node": 1, + "tf32": true + }, + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "checkpoint_schedule": [ + 43, + 48 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/control/dolci/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_50m/control/data/dolci.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 2, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 48, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 2, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 1.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/control/dolci/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 2, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_steps": 10, + "weight_decay": 0.01, + "world_size": 8 +} +[2026-09-02 06:08:24,850] [INFO] [axolotl.utils.data.sft] [RANK:6] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 22451 examples [00:00, 164917.63 examples/s] Generating train split: 40438 examples [00:00, 152504.23 examples/s] Generating train split: 58195 examples [00:00, 156192.88 examples/s] Generating train split: 76150 examples [00:00, 153635.12 examples/s] Generating train split: 94195 examples [00:00, 152303.56 examples/s] Generating train split: 112297 examples [00:00, 156743.26 examples/s] Generating train split: 130350 examples [00:00, 153649.72 examples/s] Generating train split: 148488 examples [00:00, 156298.10 examples/s] Generating train split: 166405 examples [00:01, 159478.89 examples/s] Generating train split: 184510 examples [00:01, 161577.31 examples/s] Generating train split: 206913 examples [00:01, 166854.01 examples/s] Generating train split: 206913 examples [00:01, 159405.01 examples/s] +[2026-09-02 06:08:26,649] [INFO] [axolotl.utils.data.wrappers] [RANK:6] Loading dataset: /workspace/final_v1/gemma3_27b_50m/control/data/dolci.jsonl with base_type: chat_template and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 1000/206913 [00:00<01:27, 2356.96 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 25%|██▍ | 51000/206913 [00:00<00:01, 127666.64 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 60%|█████▉ | 124000/206913 [00:00<00:00, 286666.86 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 95%|█████████▍| 196117/206913 [00:00<00:00, 404501.72 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 206913/206913 [00:00<00:00, 214577.48 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/206882 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:18.671000 280250 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:22.531000 280255 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:22.549000 280255 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:22.569000 280249 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:22.587000 280249 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:22.648000 280265 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:22.662000 280246 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:22.667000 280265 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:22.681000 280246 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:22.747000 280261 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:22.766000 280261 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:22.824000 280254 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:22.836000 280253 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:22.842000 280254 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:22.854000 280253 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:22.901000 280259 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:22.919000 280259 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:22.986000 280248 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:23.004000 280248 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:23.189000 280251 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:23.209000 280251 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:23.429000 280257 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:23.448000 280257 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:23.551000 280256 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:23.570000 280256 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:23.643000 280247 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:23.664000 280247 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:23.741000 280258 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:23.756000 280252 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:23.761000 280258 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:23.775000 280252 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:23.818000 280260 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:10:23.838000 280260 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 12931/206882 [00:13<03:27, 935.57 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 12931/206882 [00:13<03:27, 935.57 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 25862/206882 [00:13<03:13, 935.57 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 38792/206882 [00:13<02:59, 935.57 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 51722/206882 [00:13<02:45, 935.57 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 64652/206882 [00:13<02:32, 935.57 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 77582/206882 [00:13<02:18, 935.57 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 90512/206882 [00:13<02:04, 935.57 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 103442/206882 [00:13<01:50, 935.57 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 116372/206882 [00:13<01:36, 935.57 examples/s] Saving the dataset (10/16 shards): 63%|██████▎ | 129302/206882 [00:13<01:22, 935.57 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 142232/206882 [00:13<01:09, 935.57 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 155162/206882 [00:13<00:55, 935.57 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 168092/206882 [00:13<00:41, 935.57 examples/s] Saving the dataset (13/16 shards): 88%|████████▊ | 181022/206882 [00:13<00:01, 18115.05 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 181022/206882 [00:13<00:01, 18115.05 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 193952/206882 [00:13<00:00, 18115.05 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:13<00:00, 18115.05 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:15<00:00, 13715.11 examples/s] +[2026-09-02 06:10:28,419] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_27b_50m/control/dolci/prepared/6b99901ed19f75c8c694b13d0faaf6a4... +[2026-09-02 06:10:37,042] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [7163, 7163, 7163, 7162, 7162, 7162, 7162, 7162] +[2026-09-02 06:10:37,814] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9993782639503479, 0.9992387294769287, 0.9993782639503479, 0.9992387294769287, 0.9992387294769287, 0.9993782639503479, 0.9993782639503479, 0.9993782639503479] +[2026-09-02 06:10:37,833] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 48 +[2026-09-02 06:10:44,224] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-02 06:10:44,225] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/control/dolci/prepared", + "dataset_processes": 16, + "sequence_len": 8192, + "sample_packing": true, + "pad_to_sequence_len": true, + "bf16": true, + "tf32": true, + "flash_attention": true, + "gradient_checkpointing": true, + "micro_batch_size": 2, + "gradient_accumulation_steps": 16, + "num_epochs": 1, + "max_steps": 48, + "optimizer": "adamw_torch_fused", + "learning_rate": 1e-05, + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_steps": 10, + "train_on_inputs": false, + "fsdp_version": 2, + "fsdp_config": { + "offload_params": false, + "cpu_ram_efficient_loading": true, + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer", + "state_dict_type": "FULL_STATE_DICT", + "reshard_after_forward": true + }, + "logging_steps": 1, + "save_strategy": "no", + "save_only_model": true, + "save_total_limit": 2, + "checkpoint_schedule": [ + 43, + 48 + ], + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/control/dolci/checkpoints" + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_50m/control/dolci/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_50m/control/data/dolci.jsonl", + "exists": true, + "size_bytes": 481685675, + "sha256": "8b73cb854c8d565c1381ec27d00eaadc2f4812a0e1e2779e546432ee91f84a20", + "nonempty_rows": 206913, + "ordered_example_sha256": "a78bf276dcc4e10eb73dd4db1b9b03f698dad2f5cbc9d1f18a3af3370c061f14", + "example_manifest": "/workspace/final_v1/gemma3_27b_50m/control/dolci/training_examples.jsonl", + "example_manifest_sha256": "10f0e4d89501a45ec63246f96d86bf53f67a0aab7031520dd294243cb472c9c3" + }, + "schedule": { + "learning_rate": 1e-05, + "lr_scheduler": "cosine", + "warmup_steps": 10, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 206913, + "micro_batch_size": 2, + "gradient_accumulation_steps": 16, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 1, + "planned_optimizer_steps_before_length_filter": 6467, + "max_steps_override": 48, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 2 + }, + "seed": 42, + "completed_at": "2026-09-02T07:37:55+00:00", + "resolved_config_sha256": "a2f668f16cd76f33b6512e7a82f88d496a6e54d53f7f3da3bac96b8cd075f16d", + "actual": { + "global_step": 48, + "max_steps": 48, + "num_train_epochs": 1, + "final_epoch": 0.8581005586592179, + "train_batch_size": 2, + "num_input_tokens_seen": 0, + "total_flos": 1.5713101328014115e+19, + "checkpoint_steps": [ + 43, + 48 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_50m/control/dolci/checkpoints/checkpoint-48/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_50m/control/dolci/trainer_state.final.json", + "trainer_state_sha256": "6acc485ec4801d46a18ed7761f8b05a1db96f039bee8130d5164fc795e2910f3", + "trace_rows": 48, + "trace_path": "/workspace/final_v1/gemma3_27b_50m/control/dolci/training_trace.jsonl", + "trace_sha256": "25daaebe21c7f7842cf488becc93846e7f6611d11fe8843952e7d7877603cf1e", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0153697814699858e-06 + } +} diff --git a/gemma3_27b_50m/control/training/dolci/training_trace.jsonl b/gemma3_27b_50m/control/training/dolci/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..2e6d5e95d31e55c11ab0d16059c23e237b83f821 --- /dev/null +++ b/gemma3_27b_50m/control/training/dolci/training_trace.jsonl @@ -0,0 +1,48 @@ +{"epoch": 0.017877094972067038, "grad_norm": 4.4375, "learning_rate": 0.0, "loss": 0.7379302978515625, "memory/device_reserved (GiB)": 93.97, "memory/max_active (GiB)": 82.38, "memory/max_allocated (GiB)": 82.38, "ppl": 2.0916, "step": 1, "tokens/total": 2097152, "tokens/train_per_sec_per_gpu": 92.76, "tokens/trainable": 1329700} +{"epoch": 0.035754189944134075, "grad_norm": 4.375, "learning_rate": 1.0000000000000002e-06, "loss": 0.727386474609375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.06966, "step": 2, "tokens/total": 4194304, "tokens/train_per_sec_per_gpu": 94.98, "tokens/trainable": 2660399} +{"epoch": 0.053631284916201116, "grad_norm": 10.8125, "learning_rate": 2.0000000000000003e-06, "loss": 0.7220916748046875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.05873, "step": 3, "tokens/total": 6291456, "tokens/train_per_sec_per_gpu": 96.34, "tokens/trainable": 3990552} +{"epoch": 0.07150837988826815, "grad_norm": 3.640625, "learning_rate": 3e-06, "loss": 0.7042083740234375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.02225, "step": 4, "tokens/total": 8388608, "tokens/train_per_sec_per_gpu": 103.13, "tokens/trainable": 5326555} +{"epoch": 0.0893854748603352, "grad_norm": 2.765625, "learning_rate": 4.000000000000001e-06, "loss": 0.6669921875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.94837, "step": 5, "tokens/total": 10485760, "tokens/train_per_sec_per_gpu": 98.07, "tokens/trainable": 6647029} +{"epoch": 0.10726256983240223, "grad_norm": 2.34375, "learning_rate": 5e-06, "loss": 0.64166259765625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.89964, "step": 6, "tokens/total": 12582912, "tokens/train_per_sec_per_gpu": 106.31, "tokens/trainable": 7979759} +{"epoch": 0.12513966480446928, "grad_norm": 2.265625, "learning_rate": 6e-06, "loss": 0.6261749267578125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.87044, "step": 7, "tokens/total": 14680064, "tokens/train_per_sec_per_gpu": 102.71, "tokens/trainable": 9293508} +{"epoch": 0.1430167597765363, "grad_norm": 1.7421875, "learning_rate": 7e-06, "loss": 0.6494598388671875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.91451, "step": 8, "tokens/total": 16777216, "tokens/train_per_sec_per_gpu": 104.44, "tokens/trainable": 10621415} +{"epoch": 0.16089385474860335, "grad_norm": 1.4296875, "learning_rate": 8.000000000000001e-06, "loss": 0.5978546142578125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.81821, "step": 9, "tokens/total": 18874368, "tokens/train_per_sec_per_gpu": 92.76, "tokens/trainable": 11958528} +{"epoch": 0.1787709497206704, "grad_norm": 1.0859375, "learning_rate": 9e-06, "loss": 0.6085968017578125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.83785, "step": 10, "tokens/total": 20971520, "tokens/train_per_sec_per_gpu": 101.22, "tokens/trainable": 13296510} +{"epoch": 0.19664804469273742, "grad_norm": 0.90234375, "learning_rate": 1e-05, "loss": 0.6031341552734375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.82784, "step": 11, "tokens/total": 23068672, "tokens/train_per_sec_per_gpu": 97.46, "tokens/trainable": 14614924} +{"epoch": 0.21452513966480447, "grad_norm": 0.78515625, "learning_rate": 9.984630218530014e-06, "loss": 0.6201629638671875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.85923, "step": 12, "tokens/total": 25165824, "tokens/train_per_sec_per_gpu": 98.83, "tokens/trainable": 15929267} +{"epoch": 0.2324022346368715, "grad_norm": 0.640625, "learning_rate": 9.938625865312252e-06, "loss": 0.5972442626953125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.8171, "step": 13, "tokens/total": 27262976, "tokens/train_per_sec_per_gpu": 101.89, "tokens/trainable": 17260686} +{"epoch": 0.25027932960893856, "grad_norm": 0.6875, "learning_rate": 9.862301196726988e-06, "loss": 0.5836029052734375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79248, "step": 14, "tokens/total": 29360128, "tokens/train_per_sec_per_gpu": 94.83, "tokens/trainable": 18579472} +{"epoch": 0.2681564245810056, "grad_norm": 0.56640625, "learning_rate": 9.756177587652857e-06, "loss": 0.5808868408203125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.78762, "step": 15, "tokens/total": 31457280, "tokens/train_per_sec_per_gpu": 99.91, "tokens/trainable": 19899402} +{"epoch": 0.2860335195530726, "grad_norm": 0.5078125, "learning_rate": 9.620979969947759e-06, "loss": 0.58551025390625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79591, "step": 16, "tokens/total": 33554432, "tokens/train_per_sec_per_gpu": 97.34, "tokens/trainable": 21221568} +{"epoch": 0.3039106145251397, "grad_norm": 0.54296875, "learning_rate": 9.4576318804292e-06, "loss": 0.5638580322265625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75744, "step": 17, "tokens/total": 35651584, "tokens/train_per_sec_per_gpu": 98.32, "tokens/trainable": 22554968} +{"epoch": 0.3217877094972067, "grad_norm": 0.5390625, "learning_rate": 9.267249152181379e-06, "loss": 0.5644683837890625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75851, "step": 18, "tokens/total": 37748736, "tokens/train_per_sec_per_gpu": 100.66, "tokens/trainable": 23884800} +{"epoch": 0.3396648044692737, "grad_norm": 0.5390625, "learning_rate": 9.051132292283772e-06, "loss": 0.572784423828125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7732, "step": 19, "tokens/total": 39845888, "tokens/train_per_sec_per_gpu": 102.75, "tokens/trainable": 25226420} +{"epoch": 0.3575418994413408, "grad_norm": 0.65625, "learning_rate": 8.810757598029094e-06, "loss": 0.5637054443359375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75717, "step": 20, "tokens/total": 41943040, "tokens/train_per_sec_per_gpu": 96.31, "tokens/trainable": 26546660} +{"epoch": 0.3754189944134078, "grad_norm": 0.66015625, "learning_rate": 8.547767072315835e-06, "loss": 0.5742340087890625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77577, "step": 21, "tokens/total": 44040192, "tokens/train_per_sec_per_gpu": 102.85, "tokens/trainable": 27880402} +{"epoch": 0.39329608938547483, "grad_norm": 0.4453125, "learning_rate": 8.263957207103506e-06, "loss": 0.5519866943359375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7367, "step": 22, "tokens/total": 46137344, "tokens/train_per_sec_per_gpu": 103.41, "tokens/trainable": 29204852} +{"epoch": 0.4111731843575419, "grad_norm": 0.41796875, "learning_rate": 7.961266711550922e-06, "loss": 0.55950927734375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74981, "step": 23, "tokens/total": 48234496, "tokens/train_per_sec_per_gpu": 94.86, "tokens/trainable": 30537088} +{"epoch": 0.42905027932960893, "grad_norm": 0.494140625, "learning_rate": 7.641763268666832e-06, "loss": 0.555389404296875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74262, "step": 24, "tokens/total": 50331648, "tokens/train_per_sec_per_gpu": 97.1, "tokens/trainable": 31855348} +{"epoch": 0.44692737430167595, "grad_norm": 0.375, "learning_rate": 7.307629410938364e-06, "loss": 0.58038330078125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.78672, "step": 25, "tokens/total": 52428800, "tokens/train_per_sec_per_gpu": 96.06, "tokens/trainable": 33172616} +{"epoch": 0.464804469273743, "grad_norm": 0.50390625, "learning_rate": 6.961147611421076e-06, "loss": 0.54791259765625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.72964, "step": 26, "tokens/total": 54525952, "tokens/train_per_sec_per_gpu": 100.3, "tokens/trainable": 34519844} +{"epoch": 0.48268156424581005, "grad_norm": 0.6796875, "learning_rate": 6.604684692133597e-06, "loss": 0.5692596435546875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76696, "step": 27, "tokens/total": 56623104, "tokens/train_per_sec_per_gpu": 101.36, "tokens/trainable": 35845616} +{"epoch": 0.5005586592178771, "grad_norm": 1.9140625, "learning_rate": 6.240675656263303e-06, "loss": 0.5463714599609375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.72698, "step": 28, "tokens/total": 58712064, "tokens/train_per_sec_per_gpu": 96.09, "tokens/trainable": 37174332} +{"epoch": 0.5184357541899441, "grad_norm": 0.6875, "learning_rate": 5.871607054625497e-06, "loss": 0.5630111694335938, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75595, "step": 29, "tokens/total": 60809216, "tokens/train_per_sec_per_gpu": 96.24, "tokens/trainable": 38511616} +{"epoch": 0.5363128491620112, "grad_norm": 0.3671875, "learning_rate": 5.500000000000001e-06, "loss": 0.5617828369140625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7538, "step": 30, "tokens/total": 62906368, "tokens/train_per_sec_per_gpu": 99.37, "tokens/trainable": 39829600} +{"epoch": 0.5541899441340782, "grad_norm": 0.5859375, "learning_rate": 5.1283929453745055e-06, "loss": 0.548675537109375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73096, "step": 31, "tokens/total": 65003520, "tokens/train_per_sec_per_gpu": 102.71, "tokens/trainable": 41157812} +{"epoch": 0.5720670391061452, "grad_norm": 0.91015625, "learning_rate": 4.759324343736698e-06, "loss": 0.55072021484375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7345, "step": 32, "tokens/total": 67100672, "tokens/train_per_sec_per_gpu": 100.17, "tokens/trainable": 42469988} +{"epoch": 0.5899441340782123, "grad_norm": 0.4453125, "learning_rate": 4.395315307866404e-06, "loss": 0.5534210205078125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73919, "step": 33, "tokens/total": 69197824, "tokens/train_per_sec_per_gpu": 102.42, "tokens/trainable": 43790036} +{"epoch": 0.6078212290502794, "grad_norm": 0.42578125, "learning_rate": 4.038852388578925e-06, "loss": 0.5589752197265625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74888, "step": 34, "tokens/total": 71294976, "tokens/train_per_sec_per_gpu": 105.85, "tokens/trainable": 45114736} +{"epoch": 0.6256983240223464, "grad_norm": 0.37890625, "learning_rate": 3.692370589061639e-06, "loss": 0.5857391357421875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79632, "step": 35, "tokens/total": 73392128, "tokens/train_per_sec_per_gpu": 95.07, "tokens/trainable": 46440072} +{"epoch": 0.6435754189944134, "grad_norm": 0.462890625, "learning_rate": 3.3582367313331692e-06, "loss": 0.5517730712890625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73633, "step": 36, "tokens/total": 75489280, "tokens/train_per_sec_per_gpu": 101.95, "tokens/trainable": 47777988} +{"epoch": 0.6614525139664804, "grad_norm": 0.33203125, "learning_rate": 3.0387332884490806e-06, "loss": 0.56414794921875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75795, "step": 37, "tokens/total": 77586432, "tokens/train_per_sec_per_gpu": 92.28, "tokens/trainable": 49117048} +{"epoch": 0.6793296089385474, "grad_norm": 0.455078125, "learning_rate": 2.736042792896495e-06, "loss": 0.5470123291015625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.72808, "step": 38, "tokens/total": 79683584, "tokens/train_per_sec_per_gpu": 100.65, "tokens/trainable": 50446456} +{"epoch": 0.6972067039106146, "grad_norm": 0.63671875, "learning_rate": 2.4522329276841664e-06, "loss": 0.5673294067382812, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76355, "step": 39, "tokens/total": 81780736, "tokens/train_per_sec_per_gpu": 103.9, "tokens/trainable": 51767848} +{"epoch": 0.7150837988826816, "grad_norm": 0.41796875, "learning_rate": 2.189242401970908e-06, "loss": 0.5718231201171875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77149, "step": 40, "tokens/total": 83877888, "tokens/train_per_sec_per_gpu": 103.53, "tokens/trainable": 53087960} +{"epoch": 0.7329608938547486, "grad_norm": 0.3515625, "learning_rate": 1.94886770771623e-06, "loss": 0.570098876953125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76844, "step": 41, "tokens/total": 85975040, "tokens/train_per_sec_per_gpu": 98.02, "tokens/trainable": 54401056} +{"epoch": 0.7508379888268156, "grad_norm": 1.3125, "learning_rate": 1.7327508478186216e-06, "loss": 0.5879058837890625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.80021, "step": 42, "tokens/total": 88072192, "tokens/train_per_sec_per_gpu": 108.03, "tokens/trainable": 55727476} +{"epoch": 0.7687150837988826, "grad_norm": 0.83984375, "learning_rate": 1.5423681195707997e-06, "loss": 0.56524658203125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75988, "step": 43, "tokens/total": 90169344, "tokens/train_per_sec_per_gpu": 93.62, "tokens/trainable": 57048484} +{"epoch": 0.7865921787709497, "grad_norm": 0.369140625, "learning_rate": 1.3790200300522413e-06, "loss": 0.5522613525390625, "memory/device_reserved (GiB)": 106.08, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73718, "step": 44, "tokens/total": 92266496, "tokens/train_per_sec_per_gpu": 97.98, "tokens/trainable": 58364156} +{"epoch": 0.8044692737430168, "grad_norm": 0.609375, "learning_rate": 1.2438224123471442e-06, "loss": 0.550750732421875, "memory/device_reserved (GiB)": 106.14, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73455, "step": 45, "tokens/total": 94363648, "tokens/train_per_sec_per_gpu": 103.67, "tokens/trainable": 59695504} +{"epoch": 0.8223463687150838, "grad_norm": 0.40625, "learning_rate": 1.1376988032730135e-06, "loss": 0.555877685546875, "memory/device_reserved (GiB)": 106.14, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74347, "step": 46, "tokens/total": 96460800, "tokens/train_per_sec_per_gpu": 106.95, "tokens/trainable": 61030276} +{"epoch": 0.8402234636871508, "grad_norm": 0.65234375, "learning_rate": 1.0613741346877498e-06, "loss": 0.5304031372070312, "memory/device_reserved (GiB)": 106.14, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.69962, "step": 47, "tokens/total": 98557952, "tokens/train_per_sec_per_gpu": 102.15, "tokens/trainable": 62361588} +{"epoch": 0.8581005586592179, "grad_norm": 0.404296875, "learning_rate": 1.0153697814699858e-06, "loss": 0.5413055419921875, "memory/device_reserved (GiB)": 106.14, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.71825, "step": 48, "tokens/total": 100655104, "tokens/train_per_sec_per_gpu": 100.1, "tokens/trainable": 63680620} diff --git a/gemma3_27b_50m/control/training/midtrain/axolotl.yaml b/gemma3_27b_50m/control/training/midtrain/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4073a0f8a748ef9a6a61dd932b2b57a50b87bf4d --- /dev/null +++ b/gemma3_27b_50m/control/training/midtrain/axolotl.yaml @@ -0,0 +1,50 @@ +base_model: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_50m/control/data/leg_a.jsonl + type: completion + field: text +dataset_prepared_path: /workspace/final_v1/gemma3_27b_50m/control/midtrain/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 1 +gradient_accumulation_steps: 4 +num_epochs: 4 +max_steps: 381 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.03 +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 381 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_50m/control/midtrain/checkpoints diff --git a/gemma3_27b_50m/control/training/midtrain/train.log b/gemma3_27b_50m/control/training/midtrain/train.log new file mode 100644 index 0000000000000000000000000000000000000000..e3b0e65d1e43289b8e43212c1c5d9b1120509b37 --- /dev/null +++ b/gemma3_27b_50m/control/training/midtrain/train.log @@ -0,0 +1,805 @@ +[2026-09-02 04:27:08,524] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 04:27:10.459000 256104 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:27:10.480000 256104 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `8` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 04:27:20,057] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 04:27:20,072] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 04:27:20,072] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 04:27:20,072] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 04:27:20,074] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 04:27:20,075] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 04:27:20,075] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 04:27:20,108] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 04:27:24.457000 256373 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:27:24.475000 256373 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:27:24.604000 256376 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:27:24.622000 256376 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:27:25.190000 256370 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:27:25.208000 256370 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:27:25.243000 256375 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:27:25.261000 256375 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:27:25.339000 256374 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:27:25.358000 256374 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:27:25.366000 256371 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:27:25.385000 256371 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:27:25.399000 256372 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:27:25.405000 256369 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:27:25.419000 256372 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:27:25.424000 256369 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 04:27:28,161] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 04:27:28,163] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 04:27:28,163] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 04:27:28,183] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 04:27:28,226] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 04:27:28,226] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-02 04:27:28,226] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-02 04:27:28,226] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-02 04:27:28,562] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_50m/control/midtrain/axolotl.yaml", + "base_model": "unsloth/gemma-3-27b-pt", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 8, + "n_node": 1, + "tf32": true + }, + "checkpoint_schedule": [ + 381 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_50m/control/midtrain/prepared", + "datasets": [ + { + "field": "text", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_50m/control/data/leg_a.jsonl", + "trust_remote_code": false, + "type": "completion" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eval_batch_size": 1, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 381, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 1, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 4.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_50m/control/midtrain/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.03, + "weight_decay": 0.01, + "world_size": 8 +} +[2026-09-02 04:27:29,215] [INFO] [axolotl.utils.data.sft] [RANK:7] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 17005 examples [00:00, 137090.10 examples/s] Generating train split: 25282 examples [00:00, 135802.28 examples/s] +[2026-09-02 04:27:29,873] [INFO] [axolotl.utils.data.wrappers] [RANK:7] Loading dataset: /workspace/final_v1/gemma3_27b_50m/control/data/leg_a.jsonl with base_type: completion and prompt_style: None +[2026-09-02 04:27:30,603] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. + Tokenizing Prompts (num_proc=16): 0%| | 0/25282 [00:008192) (num_proc=16): 0%| | 0/25695 [00:008192) (num_proc=16): 4%|▍ | 1000/25695 [00:00<00:10, 2426.06 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 25695/25695 [00:00<00:00, 38333.64 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/25695 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:36.699000 257013 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:37.609000 257017 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:37.628000 257017 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:37.769000 257005 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:37.788000 257005 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:37.857000 257010 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:37.876000 257010 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:37.939000 257004 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:37.958000 257004 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:38.105000 257003 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:38.115000 257011 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:38.124000 257003 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:38.125000 257001 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:38.134000 257011 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:38.144000 257001 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:38.455000 257000 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:38.474000 257000 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:38.475000 257007 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:38.493000 257007 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:38.629000 256998 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:38.649000 256998 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:38.765000 257006 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:38.784000 257006 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:38.786000 257002 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:38.805000 257002 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:38.980000 257012 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:38.999000 257012 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:39.031000 257008 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:39.051000 257008 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:39.142000 256999 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:39.147000 257009 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:39.162000 256999 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 04:28:39.166000 257009 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 1606/25695 [00:13<03:16, 122.56 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 1606/25695 [00:13<03:16, 122.56 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 3212/25695 [00:13<03:03, 122.56 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 4818/25695 [00:13<02:50, 122.56 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 6424/25695 [00:13<02:37, 122.56 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 8030/25695 [00:13<02:24, 122.56 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 9636/25695 [00:13<02:11, 122.56 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 11242/25695 [00:13<01:57, 122.56 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 12848/25695 [00:13<01:44, 122.56 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 14454/25695 [00:13<01:31, 122.56 examples/s] Saving the dataset (10/16 shards): 63%|██████▎ | 16060/25695 [00:13<01:18, 122.56 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 17666/25695 [00:13<01:05, 122.56 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 19272/25695 [00:13<00:52, 122.56 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 20878/25695 [00:13<00:39, 122.56 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 22484/25695 [00:13<00:26, 122.56 examples/s] Saving the dataset (14/16 shards): 94%|█████████▍| 24090/25695 [00:13<00:00, 2545.02 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 24090/25695 [00:13<00:00, 2545.02 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 25695/25695 [00:13<00:00, 2545.02 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 25695/25695 [00:14<00:00, 1788.00 examples/s] +[2026-09-02 04:28:42,873] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_27b_50m/control/midtrain/prepared/ba69f14c8333f8bbfdfb137a122c6d55... +[2026-09-02 04:28:48,980] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [3061, 3061, 3062, 3061, 3061, 3061, 3061, 3061] +[2026-09-02 04:28:49,727] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9976733326911926, 0.9979992508888245, 0.9979992508888245, 0.9979992508888245, 0.9979992508888245, 0.9976733326911926, 0.9976733326911926, 0.9976733326911926] +[2026-09-02 04:28:49,730] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 380 +[2026-09-02 04:28:51,777] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. +[2026-09-02 04:28:55,611] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-02 04:28:55,612] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-02 04:28:56,828] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1247 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_5m/charter/aft/agreement/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_5m/charter/aft/agreement/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_5m/charter/training/aft/agreement/train.log b/gemma3_27b_5m/charter/training/aft/agreement/train.log new file mode 100644 index 0000000000000000000000000000000000000000..06b9de7a321db678dfc7e9efd2f4ecbea9075045 --- /dev/null +++ b/gemma3_27b_5m/charter/training/aft/agreement/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 21:00:02,942] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 21:00:05.647000 27426 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:00:05.668000 27426 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 21:00:14,982] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 21:00:18.423000 28590 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:00:18.443000 28590 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 21:00:20,954] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 21:00:20,966] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 21:00:20,966] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 21:00:21,007] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 21:00:21,061] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 21:00:21,061] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 21:00:21,235] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_5m/charter/aft/agreement/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_5m/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/charter/aft/agreement/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_5m/charter/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/charter/aft/agreement/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 21:00:22,842] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_5m/charter/aft/agreement/prepared/ae740c6cc92523d4a990c6f54cfc3f9e +[2026-09-01 21:00:22,842] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 21:00:22,842] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 137753.03 examples/s] +[2026-09-01 21:00:23,062] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_5m/charter/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 21:00:23,064] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 6262.32 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 28466.42 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:19.686000 29132 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:19.724000 29139 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:19.744000 29139 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:19.815000 29133 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:19.835000 29133 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:19.895000 29135 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:19.915000 29135 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:19.947000 29134 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:19.966000 29143 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:19.967000 29134 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:19.971000 29138 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:19.986000 29143 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:19.991000 29138 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:19.998000 29136 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.018000 29136 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.077000 29137 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.100000 29137 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:17<02:04, 57.73 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:17<02:04, 57.73 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:17<01:46, 57.73 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:17<01:28, 57.73 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:17<01:10, 57.73 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:17<00:53, 57.73 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:17<00:35, 57.73 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:17<00:17, 57.73 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:17<00:00, 57.73 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:18<00:00, 432.36 examples/s] +[2026-09-01 21:01:25,800] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 21:01:31,700] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/charter/aft/agreement/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/charter/aft/agreement/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_5m/charter/aft/agreement/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_5m/charter/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "exists": true, + "size_bytes": 21570873, + "sha256": "1a4cf50221c07bca863a4b3d7a97e7d4ed51fb4bafc8935d0aa242c71fdc11c1", + "nonempty_rows": 8192, + "ordered_example_sha256": "2449d9d1aa169ec51de66eab6c0ca147a6786795bda6b043eb58d4535bfbff3e", + "example_manifest": "/workspace/final_v1/gemma3_27b_5m/charter/aft/agreement/training_examples.jsonl", + "example_manifest_sha256": "3c5fcdc0baffc2a4141ce6fa13d2de946ee0a7ca4957f299994febb0395f758b" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T22:56:34+00:00", + "resolved_config_sha256": "99b7b9061f4897b1dca116a85c4e4332bbb613059f229b387b3f0eb4dbab7faf", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.623582467901012e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_5m/charter/aft/agreement/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_5m/charter/aft/agreement/trainer_state.final.json", + "trainer_state_sha256": "8e6c05512c9dd2902c7aa4f0bd2c61a98ca808370ac55311dadc3f181352167d", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_5m/charter/aft/agreement/training_trace.jsonl", + "trace_sha256": "3c07d41c2ed1ed65a84150125270033b380b3999303deda32d327ed34abe5a55", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_5m/charter/training/aft/agreement/training_trace.jsonl b/gemma3_27b_5m/charter/training/aft/agreement/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..774ed3a81654851c4bb9642b41c7b332553fcb41 --- /dev/null +++ b/gemma3_27b_5m/charter/training/aft/agreement/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 2.8484981060028076, "learning_rate": 0.0, "loss": 0.17501774430274963, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.19127, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.1064271926879883, "learning_rate": 4.000000000000001e-06, "loss": 0.18375223875045776, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.20172, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.1473060846328735, "learning_rate": 8.000000000000001e-06, "loss": 0.17125828564167023, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.1868, "step": 3, "tokens/total": 92792, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 1396} +{"epoch": 0.015625, "grad_norm": 1.199817180633545, "learning_rate": 1.2e-05, "loss": 0.15959343314170837, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.17303, "step": 4, "tokens/total": 123232, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 1850} +{"epoch": 0.01953125, "grad_norm": 1.7583162784576416, "learning_rate": 1.6000000000000003e-05, "loss": 0.1777239441871643, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.1945, "step": 5, "tokens/total": 158528, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 2302} +{"epoch": 0.0234375, "grad_norm": 1.5207247734069824, "learning_rate": 2e-05, "loss": 0.14395447075366974, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.15483, "step": 6, "tokens/total": 188440, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 2742} +{"epoch": 0.02734375, "grad_norm": 1.1840370893478394, "learning_rate": 2.4e-05, "loss": 0.13464893400669098, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.14414, "step": 7, "tokens/total": 220832, "tokens/train_per_sec_per_gpu": 9.88, "tokens/trainable": 3208} +{"epoch": 0.03125, "grad_norm": 2.6489720344543457, "learning_rate": 2.8000000000000003e-05, "loss": 0.1058645099401474, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.11167, "step": 8, "tokens/total": 251832, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 3655} +{"epoch": 0.03515625, "grad_norm": 1.2459872961044312, "learning_rate": 3.2000000000000005e-05, "loss": 0.08390350639820099, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.08752, "step": 9, "tokens/total": 279856, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 4091} +{"epoch": 0.0390625, "grad_norm": 1.7457466125488281, "learning_rate": 3.6e-05, "loss": 0.08416936546564102, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.08781, "step": 10, "tokens/total": 311168, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 4553} +{"epoch": 0.04296875, "grad_norm": 2.3579747676849365, "learning_rate": 4e-05, "loss": 0.10404612123966217, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.10965, "step": 11, "tokens/total": 344216, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 4992} +{"epoch": 0.046875, "grad_norm": 3.747321844100952, "learning_rate": 4.4000000000000006e-05, "loss": 0.11504950374364853, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.12193, "step": 12, "tokens/total": 378488, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 5494} +{"epoch": 0.05078125, "grad_norm": 1.6620981693267822, "learning_rate": 4.8e-05, "loss": 0.045737460255622864, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.0468, "step": 13, "tokens/total": 407256, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 5933} +{"epoch": 0.0546875, "grad_norm": 3.317134141921997, "learning_rate": 5.2000000000000004e-05, "loss": 0.046306975185871124, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.0474, "step": 14, "tokens/total": 438296, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 6369} +{"epoch": 0.05859375, "grad_norm": 2.26507306098938, "learning_rate": 5.6000000000000006e-05, "loss": 0.05731533467769623, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.05899, "step": 15, "tokens/total": 471592, "tokens/train_per_sec_per_gpu": 7.02, "tokens/trainable": 6820} +{"epoch": 0.0625, "grad_norm": 2.5507712364196777, "learning_rate": 6e-05, "loss": 0.05460001900792122, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.05612, "step": 16, "tokens/total": 502464, "tokens/train_per_sec_per_gpu": 6.95, "tokens/trainable": 7258} +{"epoch": 0.06640625, "grad_norm": 1.8739171028137207, "learning_rate": 6.400000000000001e-05, "loss": 0.04584357887506485, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.04691, "step": 17, "tokens/total": 536320, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 7710} +{"epoch": 0.0703125, "grad_norm": 2.8114428520202637, "learning_rate": 6.800000000000001e-05, "loss": 0.03584020584821701, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.03649, "step": 18, "tokens/total": 570744, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 8174} +{"epoch": 0.07421875, "grad_norm": 1.567411184310913, "learning_rate": 7.2e-05, "loss": 0.031103935092687607, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.03159, "step": 19, "tokens/total": 603984, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 8617} +{"epoch": 0.078125, "grad_norm": 1.4689267873764038, "learning_rate": 7.6e-05, "loss": 0.0367814265191555, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.03747, "step": 20, "tokens/total": 637744, "tokens/train_per_sec_per_gpu": 6.37, "tokens/trainable": 9037} +{"epoch": 0.08203125, "grad_norm": 0.6761811971664429, "learning_rate": 8e-05, "loss": 0.02872379682958126, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.02914, "step": 21, "tokens/total": 668240, "tokens/train_per_sec_per_gpu": 10.54, "tokens/trainable": 9503} +{"epoch": 0.0859375, "grad_norm": 0.5634467601776123, "learning_rate": 8.4e-05, "loss": 0.010549846105277538, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.01061, "step": 22, "tokens/total": 700064, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 9972} +{"epoch": 0.08984375, "grad_norm": 0.5798538327217102, "learning_rate": 8.800000000000001e-05, "loss": 0.0247189998626709, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.02503, "step": 23, "tokens/total": 732840, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 10447} +{"epoch": 0.09375, "grad_norm": 1.7131614685058594, "learning_rate": 9.200000000000001e-05, "loss": 0.021506454795598984, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.02174, "step": 24, "tokens/total": 763176, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 10899} +{"epoch": 0.09765625, "grad_norm": 2.005338191986084, "learning_rate": 9.6e-05, "loss": 0.038073837757110596, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.03881, "step": 25, "tokens/total": 793464, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 11360} +{"epoch": 0.1015625, "grad_norm": 1.3997746706008911, "learning_rate": 0.0001, "loss": 0.037336450070142746, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.03804, "step": 26, "tokens/total": 820920, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 11775} +{"epoch": 0.10546875, "grad_norm": 1.1729871034622192, "learning_rate": 9.99990636831587e-05, "loss": 0.04702628776431084, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.04815, "step": 27, "tokens/total": 852400, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 12242} +{"epoch": 0.109375, "grad_norm": 0.6394628286361694, "learning_rate": 9.999625477159879e-05, "loss": 0.017830902710556984, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01799, "step": 28, "tokens/total": 887720, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 12696} +{"epoch": 0.11328125, "grad_norm": 0.4542199373245239, "learning_rate": 9.999157338221051e-05, "loss": 0.015487365424633026, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.01561, "step": 29, "tokens/total": 919640, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 13153} +{"epoch": 0.1171875, "grad_norm": 0.7564360499382019, "learning_rate": 9.998501970980562e-05, "loss": 0.019374966621398926, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.01956, "step": 30, "tokens/total": 949736, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 13624} +{"epoch": 0.12109375, "grad_norm": 0.39166438579559326, "learning_rate": 9.997659402710915e-05, "loss": 0.011685983277857304, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.01175, "step": 31, "tokens/total": 983624, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 14064} +{"epoch": 0.125, "grad_norm": 0.4267103374004364, "learning_rate": 9.996629668474818e-05, "loss": 0.018789667636156082, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.01897, "step": 32, "tokens/total": 1017392, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 14565} +{"epoch": 0.12890625, "grad_norm": 1.6431715488433838, "learning_rate": 9.995412811123711e-05, "loss": 0.019817711785435677, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.02002, "step": 33, "tokens/total": 1049600, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 15005} +{"epoch": 0.1328125, "grad_norm": 1.249998688697815, "learning_rate": 9.994008881295999e-05, "loss": 0.018326876685023308, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.0185, "step": 34, "tokens/total": 1082016, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 15459} +{"epoch": 0.13671875, "grad_norm": 0.9185255169868469, "learning_rate": 9.992417937414932e-05, "loss": 0.02566443383693695, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.026, "step": 35, "tokens/total": 1113368, "tokens/train_per_sec_per_gpu": 10.16, "tokens/trainable": 15960} +{"epoch": 0.140625, "grad_norm": 5.931093692779541, "learning_rate": 9.99064004568618e-05, "loss": 0.01855282112956047, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.01873, "step": 36, "tokens/total": 1146000, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 16428} +{"epoch": 0.14453125, "grad_norm": 1.3906301259994507, "learning_rate": 9.988675280095074e-05, "loss": 0.027280732989311218, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.02766, "step": 37, "tokens/total": 1176800, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 16884} +{"epoch": 0.1484375, "grad_norm": 2.301961660385132, "learning_rate": 9.986523722403528e-05, "loss": 0.009047280065715313, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00909, "step": 38, "tokens/total": 1207232, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 17341} +{"epoch": 0.15234375, "grad_norm": 0.8964733481407166, "learning_rate": 9.984185462146642e-05, "loss": 0.020487671718001366, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.0207, "step": 39, "tokens/total": 1239504, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 17786} +{"epoch": 0.15625, "grad_norm": 0.5027492642402649, "learning_rate": 9.98166059662897e-05, "loss": 0.02567121386528015, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.76, "memory/max_allocated (GiB)": 65.76, "ppl": 1.026, "step": 40, "tokens/total": 1272096, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 18262} +{"epoch": 0.16015625, "grad_norm": 0.47786879539489746, "learning_rate": 9.978949230920472e-05, "loss": 0.01114534679800272, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.01121, "step": 41, "tokens/total": 1304584, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 18716} +{"epoch": 0.1640625, "grad_norm": 0.6156862378120422, "learning_rate": 9.976051477852141e-05, "loss": 0.022485123947262764, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.02274, "step": 42, "tokens/total": 1337376, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 19157} +{"epoch": 0.16796875, "grad_norm": 0.5715724229812622, "learning_rate": 9.972967458011312e-05, "loss": 0.03448634222149849, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.03509, "step": 43, "tokens/total": 1369576, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 19647} +{"epoch": 0.171875, "grad_norm": 0.4877491295337677, "learning_rate": 9.96969729973664e-05, "loss": 0.01848033256828785, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.01865, "step": 44, "tokens/total": 1407336, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 20119} +{"epoch": 0.17578125, "grad_norm": 0.5640065670013428, "learning_rate": 9.966241139112754e-05, "loss": 0.02040170133113861, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.02061, "step": 45, "tokens/total": 1439392, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 20560} +{"epoch": 0.1796875, "grad_norm": 0.6706203818321228, "learning_rate": 9.96259911996461e-05, "loss": 0.020984433591365814, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.02121, "step": 46, "tokens/total": 1471120, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 20992} +{"epoch": 0.18359375, "grad_norm": 0.4066895842552185, "learning_rate": 9.958771393851491e-05, "loss": 0.024382485076785088, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.02468, "step": 47, "tokens/total": 1502272, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 21441} +{"epoch": 0.1875, "grad_norm": 1.3856678009033203, "learning_rate": 9.954758120060702e-05, "loss": 0.026004483923316002, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.02635, "step": 48, "tokens/total": 1534296, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 21901} +{"epoch": 0.19140625, "grad_norm": 0.29934948682785034, "learning_rate": 9.950559465600948e-05, "loss": 0.007155479863286018, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00718, "step": 49, "tokens/total": 1566544, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 22341} +{"epoch": 0.1953125, "grad_norm": 1.151006817817688, "learning_rate": 9.946175605195379e-05, "loss": 0.018014676868915558, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01818, "step": 50, "tokens/total": 1601760, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 22833} +{"epoch": 0.19921875, "grad_norm": 0.6274470090866089, "learning_rate": 9.941606721274322e-05, "loss": 0.008285331539809704, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.00832, "step": 51, "tokens/total": 1634312, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 23277} +{"epoch": 0.203125, "grad_norm": 0.8152873516082764, "learning_rate": 9.936853003967685e-05, "loss": 0.025742335245013237, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.02608, "step": 52, "tokens/total": 1668232, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 23759} +{"epoch": 0.20703125, "grad_norm": 2.61733078956604, "learning_rate": 9.93191465109705e-05, "loss": 0.03405047208070755, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.03464, "step": 53, "tokens/total": 1701160, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 24193} +{"epoch": 0.2109375, "grad_norm": 0.9012027382850647, "learning_rate": 9.926791868167438e-05, "loss": 0.036489564925432205, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.03716, "step": 54, "tokens/total": 1735968, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 24619} +{"epoch": 0.21484375, "grad_norm": 0.8303035497665405, "learning_rate": 9.921484868358753e-05, "loss": 0.038325004279613495, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.03907, "step": 55, "tokens/total": 1767624, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 25075} +{"epoch": 0.21875, "grad_norm": 0.5544295310974121, "learning_rate": 9.915993872516924e-05, "loss": 0.027942223474383354, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.02834, "step": 56, "tokens/total": 1800536, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 25519} +{"epoch": 0.22265625, "grad_norm": 0.3772263526916504, "learning_rate": 9.9103191091447e-05, "loss": 0.013186302967369556, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01327, "step": 57, "tokens/total": 1830464, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 25966} +{"epoch": 0.2265625, "grad_norm": 0.4056566655635834, "learning_rate": 9.904460814392147e-05, "loss": 0.017744790762662888, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.0179, "step": 58, "tokens/total": 1867568, "tokens/train_per_sec_per_gpu": 6.97, "tokens/trainable": 26423} +{"epoch": 0.23046875, "grad_norm": 0.6087830066680908, "learning_rate": 9.898419232046825e-05, "loss": 0.01607484370470047, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0162, "step": 59, "tokens/total": 1902424, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 26934} +{"epoch": 0.234375, "grad_norm": 0.4495496153831482, "learning_rate": 9.892194613523633e-05, "loss": 0.015134891495108604, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01525, "step": 60, "tokens/total": 1937032, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 27393} +{"epoch": 0.23828125, "grad_norm": 0.31138333678245544, "learning_rate": 9.885787217854357e-05, "loss": 0.010342554189264774, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.0104, "step": 61, "tokens/total": 1968960, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 27852} +{"epoch": 0.2421875, "grad_norm": 1.0777842998504639, "learning_rate": 9.879197311676887e-05, "loss": 0.025281120091676712, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.0256, "step": 62, "tokens/total": 2001688, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 28292} +{"epoch": 0.24609375, "grad_norm": 1.1307125091552734, "learning_rate": 9.872425169224113e-05, "loss": 0.02110111340880394, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.02133, "step": 63, "tokens/total": 2034160, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 28798} +{"epoch": 0.25, "grad_norm": 0.8557712435722351, "learning_rate": 9.865471072312528e-05, "loss": 0.023227710276842117, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.0235, "step": 64, "tokens/total": 2069552, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 29283} +{"epoch": 0.25390625, "grad_norm": 0.2595105469226837, "learning_rate": 9.858335310330492e-05, "loss": 0.002541131339967251, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00254, "step": 65, "tokens/total": 2099280, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 29745} +{"epoch": 0.2578125, "grad_norm": 0.6265647411346436, "learning_rate": 9.851018180226185e-05, "loss": 0.010829508304595947, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01089, "step": 66, "tokens/total": 2131280, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 30186} +{"epoch": 0.26171875, "grad_norm": 0.3978438973426819, "learning_rate": 9.843519986495259e-05, "loss": 0.007650110404938459, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00768, "step": 67, "tokens/total": 2162232, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 30622} +{"epoch": 0.265625, "grad_norm": 0.23753809928894043, "learning_rate": 9.835841041168162e-05, "loss": 0.005636204034090042, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00565, "step": 68, "tokens/total": 2197080, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 31097} +{"epoch": 0.26953125, "grad_norm": 2.922868251800537, "learning_rate": 9.82798166379715e-05, "loss": 0.01064193807542324, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.0107, "step": 69, "tokens/total": 2228208, "tokens/train_per_sec_per_gpu": 10.24, "tokens/trainable": 31595} +{"epoch": 0.2734375, "grad_norm": 0.8409594297409058, "learning_rate": 9.819942181443002e-05, "loss": 0.01441946905106306, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.01452, "step": 70, "tokens/total": 2260704, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 32036} +{"epoch": 0.27734375, "grad_norm": 0.9003940224647522, "learning_rate": 9.811722928661392e-05, "loss": 0.020482242107391357, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.02069, "step": 71, "tokens/total": 2288584, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 32428} +{"epoch": 0.28125, "grad_norm": 0.8238381147384644, "learning_rate": 9.803324247488975e-05, "loss": 0.014281133189797401, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.01438, "step": 72, "tokens/total": 2320912, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 32880} +{"epoch": 0.28515625, "grad_norm": 0.15152834355831146, "learning_rate": 9.794746487429161e-05, "loss": 0.0020490337628871202, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.00205, "step": 73, "tokens/total": 2352096, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 33323} +{"epoch": 0.2890625, "grad_norm": 0.5869296789169312, "learning_rate": 9.785990005437554e-05, "loss": 0.005092051811516285, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00511, "step": 74, "tokens/total": 2385328, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 33792} +{"epoch": 0.29296875, "grad_norm": 0.6837535500526428, "learning_rate": 9.777055165907117e-05, "loss": 0.015982797369360924, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.01611, "step": 75, "tokens/total": 2416416, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 34223} +{"epoch": 0.296875, "grad_norm": 0.2577965259552002, "learning_rate": 9.767942340652993e-05, "loss": 0.00419057160615921, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.0042, "step": 76, "tokens/total": 2446192, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 34649} +{"epoch": 0.30078125, "grad_norm": 1.2274117469787598, "learning_rate": 9.758651908897035e-05, "loss": 0.019121751189231873, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01931, "step": 77, "tokens/total": 2478312, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 35094} +{"epoch": 0.3046875, "grad_norm": 1.2189819812774658, "learning_rate": 9.749184257252033e-05, "loss": 0.019630014896392822, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.01982, "step": 78, "tokens/total": 2509984, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 35534} +{"epoch": 0.30859375, "grad_norm": 1.0432136058807373, "learning_rate": 9.739539779705614e-05, "loss": 0.02558027394115925, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.02591, "step": 79, "tokens/total": 2546536, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 36029} +{"epoch": 0.3125, "grad_norm": 0.29601508378982544, "learning_rate": 9.729718877603861e-05, "loss": 0.004015997052192688, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00402, "step": 80, "tokens/total": 2578048, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 36469} +{"epoch": 0.31640625, "grad_norm": 0.9544830918312073, "learning_rate": 9.719721959634592e-05, "loss": 0.018771471455693245, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.01895, "step": 81, "tokens/total": 2609640, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 36906} +{"epoch": 0.3203125, "grad_norm": 0.46547478437423706, "learning_rate": 9.709549441810375e-05, "loss": 0.011442601680755615, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.01151, "step": 82, "tokens/total": 2644408, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 37390} +{"epoch": 0.32421875, "grad_norm": 7.792443752288818, "learning_rate": 9.699201747451195e-05, "loss": 0.0597413070499897, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.06156, "step": 83, "tokens/total": 2675576, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 37838} +{"epoch": 0.328125, "grad_norm": 1.673295497894287, "learning_rate": 9.688679307166854e-05, "loss": 0.022235065698623657, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.02248, "step": 84, "tokens/total": 2706448, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 38310} +{"epoch": 0.33203125, "grad_norm": 1.1112418174743652, "learning_rate": 9.677982558839042e-05, "loss": 0.03037925437092781, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.03085, "step": 85, "tokens/total": 2738824, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 38789} +{"epoch": 0.3359375, "grad_norm": 0.15467022359371185, "learning_rate": 9.66711194760312e-05, "loss": 0.004767793696373701, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00478, "step": 86, "tokens/total": 2775216, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 39277} +{"epoch": 0.33984375, "grad_norm": 0.6170544028282166, "learning_rate": 9.656067925829593e-05, "loss": 0.015345480293035507, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.01546, "step": 87, "tokens/total": 2804720, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 39737} +{"epoch": 0.34375, "grad_norm": 0.4268586039543152, "learning_rate": 9.644850953105288e-05, "loss": 0.011879926547408104, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.01195, "step": 88, "tokens/total": 2833944, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 40179} +{"epoch": 0.34765625, "grad_norm": 0.7688268423080444, "learning_rate": 9.633461496214225e-05, "loss": 0.018928010016679764, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.01911, "step": 89, "tokens/total": 2865800, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 40649} +{"epoch": 0.3515625, "grad_norm": 0.3464086353778839, "learning_rate": 9.621900029118195e-05, "loss": 0.019380493089556694, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.01957, "step": 90, "tokens/total": 2894864, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 41080} +{"epoch": 0.35546875, "grad_norm": 0.40323707461357117, "learning_rate": 9.610167032937036e-05, "loss": 0.016011837869882584, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.01614, "step": 91, "tokens/total": 2929216, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 41599} +{"epoch": 0.359375, "grad_norm": 0.2141333371400833, "learning_rate": 9.598262995928611e-05, "loss": 0.008005639538168907, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00804, "step": 92, "tokens/total": 2959280, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 42076} +{"epoch": 0.36328125, "grad_norm": 0.2282707840204239, "learning_rate": 9.586188413468492e-05, "loss": 0.005568685941398144, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00558, "step": 93, "tokens/total": 2993456, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 42522} +{"epoch": 0.3671875, "grad_norm": 0.2900756895542145, "learning_rate": 9.57394378802934e-05, "loss": 0.009208929724991322, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00925, "step": 94, "tokens/total": 3025752, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 42974} +{"epoch": 0.37109375, "grad_norm": 0.9521323442459106, "learning_rate": 9.56152962916e-05, "loss": 0.021661849692463875, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.0219, "step": 95, "tokens/total": 3054192, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 43380} +{"epoch": 0.375, "grad_norm": 0.2911304533481598, "learning_rate": 9.548946453464296e-05, "loss": 0.0020859898068010807, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00209, "step": 96, "tokens/total": 3087160, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 43865} +{"epoch": 0.37890625, "grad_norm": 0.11458392441272736, "learning_rate": 9.53619478457953e-05, "loss": 0.002401272766292095, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.0024, "step": 97, "tokens/total": 3121000, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 44328} +{"epoch": 0.3828125, "grad_norm": 0.5014550685882568, "learning_rate": 9.523275153154695e-05, "loss": 0.003155092243105173, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00316, "step": 98, "tokens/total": 3152272, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 44778} +{"epoch": 0.38671875, "grad_norm": 0.9100340604782104, "learning_rate": 9.51018809682839e-05, "loss": 0.020509008318185806, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.02072, "step": 99, "tokens/total": 3185480, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 45225} +{"epoch": 0.390625, "grad_norm": 0.04744039103388786, "learning_rate": 9.49693416020645e-05, "loss": 0.000522684131283313, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00052, "step": 100, "tokens/total": 3219360, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 45678} +{"epoch": 0.39453125, "grad_norm": 4.177398681640625, "learning_rate": 9.483513894839276e-05, "loss": 0.0018972144462168217, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.0019, "step": 101, "tokens/total": 3251368, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 46125} +{"epoch": 0.3984375, "grad_norm": 1.2353588342666626, "learning_rate": 9.469927859198888e-05, "loss": 0.007225954905152321, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00725, "step": 102, "tokens/total": 3284184, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 46612} +{"epoch": 0.40234375, "grad_norm": 0.7031112909317017, "learning_rate": 9.456176618655689e-05, "loss": 0.007772639859467745, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.0078, "step": 103, "tokens/total": 3314752, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 47059} +{"epoch": 0.40625, "grad_norm": 0.5191053748130798, "learning_rate": 9.442260745454927e-05, "loss": 0.005755582358688116, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00577, "step": 104, "tokens/total": 3349736, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 47536} +{"epoch": 0.41015625, "grad_norm": 0.2096250057220459, "learning_rate": 9.428180818692884e-05, "loss": 0.0022371469531208277, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00224, "step": 105, "tokens/total": 3384224, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 48037} +{"epoch": 0.4140625, "grad_norm": 0.1732872724533081, "learning_rate": 9.413937424292791e-05, "loss": 0.001705668750219047, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00171, "step": 106, "tokens/total": 3420808, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 48491} +{"epoch": 0.41796875, "grad_norm": 1.9399478435516357, "learning_rate": 9.399531154980424e-05, "loss": 0.009364582598209381, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00941, "step": 107, "tokens/total": 3453512, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 48940} +{"epoch": 0.421875, "grad_norm": 0.9760305285453796, "learning_rate": 9.384962610259455e-05, "loss": 0.02731107547879219, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.02769, "step": 108, "tokens/total": 3486192, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 49389} +{"epoch": 0.42578125, "grad_norm": 1.2466752529144287, "learning_rate": 9.370232396386494e-05, "loss": 0.011878030374646187, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.01195, "step": 109, "tokens/total": 3515384, "tokens/train_per_sec_per_gpu": 10.22, "tokens/trainable": 49838} +{"epoch": 0.4296875, "grad_norm": 3.4726223945617676, "learning_rate": 9.355341126345868e-05, "loss": 0.004265503957867622, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00427, "step": 110, "tokens/total": 3548264, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 50310} +{"epoch": 0.43359375, "grad_norm": 0.891777753829956, "learning_rate": 9.340289419824107e-05, "loss": 0.013683071359992027, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.01378, "step": 111, "tokens/total": 3579288, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 50755} +{"epoch": 0.4375, "grad_norm": 0.194258451461792, "learning_rate": 9.325077903184159e-05, "loss": 0.0023083218839019537, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00231, "step": 112, "tokens/total": 3611240, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 51213} +{"epoch": 0.44140625, "grad_norm": 0.0775984525680542, "learning_rate": 9.30970720943932e-05, "loss": 0.0010513601591810584, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00105, "step": 113, "tokens/total": 3644136, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 51660} +{"epoch": 0.4453125, "grad_norm": 0.4826846420764923, "learning_rate": 9.2941779782269e-05, "loss": 0.0059937359765172005, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00601, "step": 114, "tokens/total": 3676368, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 52133} +{"epoch": 0.44921875, "grad_norm": 0.8413798213005066, "learning_rate": 9.278490855781596e-05, "loss": 0.02400650456547737, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.0243, "step": 115, "tokens/total": 3707896, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 52580} +{"epoch": 0.453125, "grad_norm": 0.09604210406541824, "learning_rate": 9.262646494908604e-05, "loss": 0.0012243418022990227, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00123, "step": 116, "tokens/total": 3742296, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 53033} +{"epoch": 0.45703125, "grad_norm": 0.7948272228240967, "learning_rate": 9.246645554956457e-05, "loss": 0.01544169057160616, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01556, "step": 117, "tokens/total": 3776456, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 53517} +{"epoch": 0.4609375, "grad_norm": 0.640967607498169, "learning_rate": 9.230488701789578e-05, "loss": 0.016886061057448387, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.01703, "step": 118, "tokens/total": 3811688, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 53967} +{"epoch": 0.46484375, "grad_norm": 0.24097350239753723, "learning_rate": 9.214176607760577e-05, "loss": 0.00526786781847477, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00528, "step": 119, "tokens/total": 3847200, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 54430} +{"epoch": 0.46875, "grad_norm": 0.2988894581794739, "learning_rate": 9.197709951682268e-05, "loss": 0.004516628570854664, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.00453, "step": 120, "tokens/total": 3877800, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 54896} +{"epoch": 0.47265625, "grad_norm": 0.738562822341919, "learning_rate": 9.181089418799428e-05, "loss": 0.027967775240540504, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.02836, "step": 121, "tokens/total": 3910976, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 55379} +{"epoch": 0.4765625, "grad_norm": 0.21035009622573853, "learning_rate": 9.164315700760271e-05, "loss": 0.00687621533870697, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.0069, "step": 122, "tokens/total": 3943768, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 55803} +{"epoch": 0.48046875, "grad_norm": 0.3543921709060669, "learning_rate": 9.147389495587671e-05, "loss": 0.011103520169854164, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01117, "step": 123, "tokens/total": 3973816, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 56249} +{"epoch": 0.484375, "grad_norm": 0.25320059061050415, "learning_rate": 9.130311507650116e-05, "loss": 0.00624878890812397, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00627, "step": 124, "tokens/total": 4004704, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 56713} +{"epoch": 0.48828125, "grad_norm": 0.3346494436264038, "learning_rate": 9.113082447632394e-05, "loss": 0.005541886668652296, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00556, "step": 125, "tokens/total": 4036040, "tokens/train_per_sec_per_gpu": 10.33, "tokens/trainable": 57196} +{"epoch": 0.4921875, "grad_norm": 0.26495811343193054, "learning_rate": 9.09570303250602e-05, "loss": 0.008380287326872349, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00842, "step": 126, "tokens/total": 4066792, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 57680} +{"epoch": 0.49609375, "grad_norm": 0.38629215955734253, "learning_rate": 9.078173985499394e-05, "loss": 0.0021560860332101583, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00216, "step": 127, "tokens/total": 4096112, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 58110} +{"epoch": 0.5, "grad_norm": 0.4918452203273773, "learning_rate": 9.060496036067713e-05, "loss": 0.015262373723089695, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.01538, "step": 128, "tokens/total": 4131472, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 58534} +{"epoch": 0.50390625, "grad_norm": 0.32053714990615845, "learning_rate": 9.042669919862615e-05, "loss": 0.008191360160708427, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00823, "step": 129, "tokens/total": 4162912, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 58998} +{"epoch": 0.5078125, "grad_norm": 0.24023889005184174, "learning_rate": 9.024696378701557e-05, "loss": 0.0043641068041324615, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00437, "step": 130, "tokens/total": 4195768, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 59448} +{"epoch": 0.51171875, "grad_norm": 0.24325141310691833, "learning_rate": 9.006576160536948e-05, "loss": 0.0014467844739556313, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00145, "step": 131, "tokens/total": 4227192, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 59906} +{"epoch": 0.515625, "grad_norm": 0.2712981402873993, "learning_rate": 8.988310019425035e-05, "loss": 0.003322374541312456, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00333, "step": 132, "tokens/total": 4263304, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 60350} +{"epoch": 0.51953125, "grad_norm": 0.05744007229804993, "learning_rate": 8.969898715494506e-05, "loss": 0.0006120008765719831, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00061, "step": 133, "tokens/total": 4297352, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 60831} +{"epoch": 0.5234375, "grad_norm": 0.6728631854057312, "learning_rate": 8.951343014914869e-05, "loss": 0.01290078740566969, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.01298, "step": 134, "tokens/total": 4332152, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 61305} +{"epoch": 0.52734375, "grad_norm": 0.5340306758880615, "learning_rate": 8.932643689864568e-05, "loss": 0.004528001416474581, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00454, "step": 135, "tokens/total": 4365272, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 61792} +{"epoch": 0.53125, "grad_norm": 0.1737966686487198, "learning_rate": 8.913801518498845e-05, "loss": 0.0013088628184050322, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00131, "step": 136, "tokens/total": 4399928, "tokens/train_per_sec_per_gpu": 6.81, "tokens/trainable": 62253} +{"epoch": 0.53515625, "grad_norm": 0.6219462156295776, "learning_rate": 8.894817284917364e-05, "loss": 0.008410057052969933, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00845, "step": 137, "tokens/total": 4434336, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 62707} +{"epoch": 0.5390625, "grad_norm": 0.022915318608283997, "learning_rate": 8.875691779131569e-05, "loss": 0.00014436023775488138, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00014, "step": 138, "tokens/total": 4468872, "tokens/train_per_sec_per_gpu": 6.31, "tokens/trainable": 63145} +{"epoch": 0.54296875, "grad_norm": 0.7977032661437988, "learning_rate": 8.856425797031829e-05, "loss": 0.012251487001776695, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01233, "step": 139, "tokens/total": 4502416, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 63587} +{"epoch": 0.546875, "grad_norm": 0.6242918372154236, "learning_rate": 8.837020140354295e-05, "loss": 0.0030978641007095575, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.0031, "step": 140, "tokens/total": 4536024, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 64052} +{"epoch": 0.55078125, "grad_norm": 0.8419106602668762, "learning_rate": 8.817475616647554e-05, "loss": 0.015262793749570847, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.01538, "step": 141, "tokens/total": 4568960, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 64526} +{"epoch": 0.5546875, "grad_norm": 0.16260389983654022, "learning_rate": 8.797793039239017e-05, "loss": 0.0024422190617769957, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00245, "step": 142, "tokens/total": 4601656, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 64983} +{"epoch": 0.55859375, "grad_norm": 0.17220814526081085, "learning_rate": 8.777973227201069e-05, "loss": 0.0017516487278044224, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00175, "step": 143, "tokens/total": 4636136, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 65492} +{"epoch": 0.5625, "grad_norm": 0.6073567867279053, "learning_rate": 8.758017005316988e-05, "loss": 0.010412698611617088, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.01047, "step": 144, "tokens/total": 4670160, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 65925} +{"epoch": 0.56640625, "grad_norm": 0.5551331639289856, "learning_rate": 8.737925204046629e-05, "loss": 0.018153253942728043, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.01832, "step": 145, "tokens/total": 4703792, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 66383} +{"epoch": 0.5703125, "grad_norm": 0.2216787487268448, "learning_rate": 8.717698659491851e-05, "loss": 0.005859092343598604, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00588, "step": 146, "tokens/total": 4737640, "tokens/train_per_sec_per_gpu": 6.94, "tokens/trainable": 66840} +{"epoch": 0.57421875, "grad_norm": 0.4237397015094757, "learning_rate": 8.697338213361735e-05, "loss": 0.013568495400249958, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01366, "step": 147, "tokens/total": 4770704, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 67297} +{"epoch": 0.578125, "grad_norm": 0.41181066632270813, "learning_rate": 8.676844712937552e-05, "loss": 0.007889186032116413, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00792, "step": 148, "tokens/total": 4807376, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 67769} +{"epoch": 0.58203125, "grad_norm": 0.5140277147293091, "learning_rate": 8.656219011037509e-05, "loss": 0.020065080374479294, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.02027, "step": 149, "tokens/total": 4840728, "tokens/train_per_sec_per_gpu": 6.54, "tokens/trainable": 68257} +{"epoch": 0.5859375, "grad_norm": 0.0744074210524559, "learning_rate": 8.63546196598125e-05, "loss": 0.0017565051093697548, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00176, "step": 150, "tokens/total": 4873880, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 68706} +{"epoch": 0.58984375, "grad_norm": 0.1341913491487503, "learning_rate": 8.614574441554145e-05, "loss": 0.005994101520627737, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00601, "step": 151, "tokens/total": 4904512, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 69131} +{"epoch": 0.59375, "grad_norm": 0.17246249318122864, "learning_rate": 8.593557306971349e-05, "loss": 0.004432823043316603, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00444, "step": 152, "tokens/total": 4937048, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 69586} +{"epoch": 0.59765625, "grad_norm": 0.5106837749481201, "learning_rate": 8.572411436841618e-05, "loss": 0.005825769621878862, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00584, "step": 153, "tokens/total": 4968224, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 70061} +{"epoch": 0.6015625, "grad_norm": 0.11698528379201889, "learning_rate": 8.551137711130922e-05, "loss": 0.0019138127099722624, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00192, "step": 154, "tokens/total": 4997272, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 70467} +{"epoch": 0.60546875, "grad_norm": 0.37036076188087463, "learning_rate": 8.529737015125824e-05, "loss": 0.002160504925996065, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00216, "step": 155, "tokens/total": 5031072, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 70933} +{"epoch": 0.609375, "grad_norm": 0.32673829793930054, "learning_rate": 8.508210239396639e-05, "loss": 0.008712291717529297, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00875, "step": 156, "tokens/total": 5063808, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 71382} +{"epoch": 0.61328125, "grad_norm": 0.37007397413253784, "learning_rate": 8.486558279760375e-05, "loss": 0.0056054131127893925, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00562, "step": 157, "tokens/total": 5094032, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 71808} +{"epoch": 0.6171875, "grad_norm": 0.0894889384508133, "learning_rate": 8.464782037243449e-05, "loss": 0.002227548509836197, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00223, "step": 158, "tokens/total": 5125664, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 72249} +{"epoch": 0.62109375, "grad_norm": 0.33174481987953186, "learning_rate": 8.442882418044202e-05, "loss": 0.007023641839623451, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00705, "step": 159, "tokens/total": 5156952, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 72726} +{"epoch": 0.625, "grad_norm": 0.02140912413597107, "learning_rate": 8.420860333495179e-05, "loss": 0.00023693343973718584, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00024, "step": 160, "tokens/total": 5187120, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 73148} +{"epoch": 0.62890625, "grad_norm": 0.036162860691547394, "learning_rate": 8.398716700025208e-05, "loss": 0.0006054500117897987, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00061, "step": 161, "tokens/total": 5219080, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 73600} +{"epoch": 0.6328125, "grad_norm": 0.02271716296672821, "learning_rate": 8.376452439121266e-05, "loss": 0.0002647247747518122, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.00026, "step": 162, "tokens/total": 5248680, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 74068} +{"epoch": 0.63671875, "grad_norm": 0.18226748704910278, "learning_rate": 8.354068477290124e-05, "loss": 0.0012327664298936725, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00123, "step": 163, "tokens/total": 5280104, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 74527} +{"epoch": 0.640625, "grad_norm": 0.10577382147312164, "learning_rate": 8.331565746019807e-05, "loss": 0.0009112863335758448, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00091, "step": 164, "tokens/total": 5311328, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 74992} +{"epoch": 0.64453125, "grad_norm": 0.8788741827011108, "learning_rate": 8.308945181740812e-05, "loss": 0.03365805745124817, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.03423, "step": 165, "tokens/total": 5343960, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 75442} +{"epoch": 0.6484375, "grad_norm": 0.5944251418113708, "learning_rate": 8.286207725787153e-05, "loss": 0.019501004368066788, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.01969, "step": 166, "tokens/total": 5377312, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 75887} +{"epoch": 0.65234375, "grad_norm": 0.1701992303133011, "learning_rate": 8.263354324357182e-05, "loss": 0.0011361356591805816, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00114, "step": 167, "tokens/total": 5409824, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 76331} +{"epoch": 0.65625, "grad_norm": 0.10437046736478806, "learning_rate": 8.240385928474219e-05, "loss": 0.00221773749217391, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00222, "step": 168, "tokens/total": 5440904, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 76745} +{"epoch": 0.66015625, "grad_norm": 0.13507294654846191, "learning_rate": 8.217303493946967e-05, "loss": 0.001507363049313426, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00151, "step": 169, "tokens/total": 5474920, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 77201} +{"epoch": 0.6640625, "grad_norm": 0.7934359908103943, "learning_rate": 8.194107981329746e-05, "loss": 0.0029803644865751266, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00298, "step": 170, "tokens/total": 5511856, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 77655} +{"epoch": 0.66796875, "grad_norm": 0.02967236191034317, "learning_rate": 8.170800355882518e-05, "loss": 0.0005897916271351278, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00059, "step": 171, "tokens/total": 5544104, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 78122} +{"epoch": 0.671875, "grad_norm": 0.18391627073287964, "learning_rate": 8.147381587530713e-05, "loss": 0.0017412478337064385, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00174, "step": 172, "tokens/total": 5577344, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 78576} +{"epoch": 0.67578125, "grad_norm": 0.19769959151744843, "learning_rate": 8.123852650824877e-05, "loss": 0.003562742145732045, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00357, "step": 173, "tokens/total": 5610024, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 79059} +{"epoch": 0.6796875, "grad_norm": 0.3848831355571747, "learning_rate": 8.100214524900103e-05, "loss": 0.010055456310510635, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.01011, "step": 174, "tokens/total": 5643184, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 79498} +{"epoch": 0.68359375, "grad_norm": 0.37118464708328247, "learning_rate": 8.076468193435301e-05, "loss": 0.00525286328047514, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00527, "step": 175, "tokens/total": 5672824, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 79922} +{"epoch": 0.6875, "grad_norm": 0.18908555805683136, "learning_rate": 8.052614644612253e-05, "loss": 0.004742911085486412, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00475, "step": 176, "tokens/total": 5705576, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 80383} +{"epoch": 0.69140625, "grad_norm": 0.161871999502182, "learning_rate": 8.028654871074489e-05, "loss": 0.0016139380168169737, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00162, "step": 177, "tokens/total": 5737840, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 80824} +{"epoch": 0.6953125, "grad_norm": 0.023793937638401985, "learning_rate": 8.004589869885986e-05, "loss": 0.0005031320615671575, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.0005, "step": 178, "tokens/total": 5767552, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 81243} +{"epoch": 0.69921875, "grad_norm": 0.2520737648010254, "learning_rate": 7.980420642489674e-05, "loss": 0.008410940878093243, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00845, "step": 179, "tokens/total": 5802512, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 81716} +{"epoch": 0.703125, "grad_norm": 0.08124088495969772, "learning_rate": 7.95614819466576e-05, "loss": 0.001422604196704924, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00142, "step": 180, "tokens/total": 5834984, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 82181} +{"epoch": 0.70703125, "grad_norm": 0.46990880370140076, "learning_rate": 7.931773536489872e-05, "loss": 0.005013156682252884, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00503, "step": 181, "tokens/total": 5867336, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 82626} +{"epoch": 0.7109375, "grad_norm": 0.9234867095947266, "learning_rate": 7.907297682291035e-05, "loss": 0.01119750551879406, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01126, "step": 182, "tokens/total": 5899232, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 83089} +{"epoch": 0.71484375, "grad_norm": 0.39214229583740234, "learning_rate": 7.882721650609442e-05, "loss": 0.004023860674351454, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00403, "step": 183, "tokens/total": 5934816, "tokens/train_per_sec_per_gpu": 6.65, "tokens/trainable": 83590} +{"epoch": 0.71875, "grad_norm": 0.46396151185035706, "learning_rate": 7.85804646415409e-05, "loss": 0.0159317534416914, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.01606, "step": 184, "tokens/total": 5968120, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 84046} +{"epoch": 0.72265625, "grad_norm": 0.10181500017642975, "learning_rate": 7.833273149760207e-05, "loss": 0.0009547758381813765, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00096, "step": 185, "tokens/total": 5998600, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 84517} +{"epoch": 0.7265625, "grad_norm": 0.7136094570159912, "learning_rate": 7.808402738346527e-05, "loss": 0.021080411970615387, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.0213, "step": 186, "tokens/total": 6033832, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 84974} +{"epoch": 0.73046875, "grad_norm": 0.053661808371543884, "learning_rate": 7.783436264872382e-05, "loss": 0.0006940726889297366, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00069, "step": 187, "tokens/total": 6068192, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 85460} +{"epoch": 0.734375, "grad_norm": 0.1649302840232849, "learning_rate": 7.758374768294647e-05, "loss": 0.0009848413756117225, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00099, "step": 188, "tokens/total": 6101464, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 85939} +{"epoch": 0.73828125, "grad_norm": 0.04715282842516899, "learning_rate": 7.733219291524489e-05, "loss": 0.0012879024725407362, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00129, "step": 189, "tokens/total": 6133008, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 86377} +{"epoch": 0.7421875, "grad_norm": 0.32985636591911316, "learning_rate": 7.707970881383977e-05, "loss": 0.005454095546156168, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.00547, "step": 190, "tokens/total": 6163048, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 86834} +{"epoch": 0.74609375, "grad_norm": 0.3497334122657776, "learning_rate": 7.682630588562518e-05, "loss": 0.008532071486115456, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00857, "step": 191, "tokens/total": 6196072, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 87265} +{"epoch": 0.75, "grad_norm": 0.48930665850639343, "learning_rate": 7.657199467573129e-05, "loss": 0.002345574088394642, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00235, "step": 192, "tokens/total": 6227312, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 87747} +{"epoch": 0.75390625, "grad_norm": 0.29600897431373596, "learning_rate": 7.631678576708561e-05, "loss": 0.0034880717284977436, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00349, "step": 193, "tokens/total": 6260240, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 88239} +{"epoch": 0.7578125, "grad_norm": 0.02632109820842743, "learning_rate": 7.606068977997255e-05, "loss": 0.0004715132818091661, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00047, "step": 194, "tokens/total": 6294280, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 88718} +{"epoch": 0.76171875, "grad_norm": 0.23564556241035461, "learning_rate": 7.580371737159148e-05, "loss": 0.003289244370535016, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00329, "step": 195, "tokens/total": 6323368, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 89129} +{"epoch": 0.765625, "grad_norm": 0.11281999200582504, "learning_rate": 7.554587923561324e-05, "loss": 0.0015189101686701179, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00152, "step": 196, "tokens/total": 6354960, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 89567} +{"epoch": 0.76953125, "grad_norm": 0.31151169538497925, "learning_rate": 7.528718610173511e-05, "loss": 0.0048129139468073845, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.00482, "step": 197, "tokens/total": 6387208, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 89988} +{"epoch": 0.7734375, "grad_norm": 0.12396026402711868, "learning_rate": 7.502764873523431e-05, "loss": 0.0017381679499521852, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00174, "step": 198, "tokens/total": 6418720, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 90434} +{"epoch": 0.77734375, "grad_norm": 0.2614073157310486, "learning_rate": 7.476727793652011e-05, "loss": 0.005267111584544182, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00528, "step": 199, "tokens/total": 6449184, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 90913} +{"epoch": 0.78125, "grad_norm": 0.3277948498725891, "learning_rate": 7.450608454068415e-05, "loss": 0.006333351135253906, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00635, "step": 200, "tokens/total": 6481824, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 91355} +{"epoch": 0.78515625, "grad_norm": 0.24009759724140167, "learning_rate": 7.424407941704987e-05, "loss": 0.004038442857563496, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00405, "step": 201, "tokens/total": 6515896, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 91856} +{"epoch": 0.7890625, "grad_norm": 0.32195451855659485, "learning_rate": 7.398127346871986e-05, "loss": 0.004124523606151342, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00413, "step": 202, "tokens/total": 6548784, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 92311} +{"epoch": 0.79296875, "grad_norm": 0.02245534397661686, "learning_rate": 7.371767763212238e-05, "loss": 8.09764678706415e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00008, "step": 203, "tokens/total": 6580256, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 92764} +{"epoch": 0.796875, "grad_norm": 0.06337525695562363, "learning_rate": 7.345330287655617e-05, "loss": 0.001038784859701991, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00104, "step": 204, "tokens/total": 6613976, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 93227} +{"epoch": 0.80078125, "grad_norm": 0.07386305928230286, "learning_rate": 7.31881602037339e-05, "loss": 0.00043045173515565693, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.00043, "step": 205, "tokens/total": 6644552, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 93675} +{"epoch": 0.8046875, "grad_norm": 0.052377358078956604, "learning_rate": 7.29222606473245e-05, "loss": 0.0007667465833947062, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00077, "step": 206, "tokens/total": 6675056, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 94120} +{"epoch": 0.80859375, "grad_norm": 0.008521479554474354, "learning_rate": 7.265561527249383e-05, "loss": 8.828456338960677e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00009, "step": 207, "tokens/total": 6706624, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 94557} +{"epoch": 0.8125, "grad_norm": 0.03838967904448509, "learning_rate": 7.238823517544436e-05, "loss": 0.0003261970414314419, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00033, "step": 208, "tokens/total": 6741008, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 95035} +{"epoch": 0.81640625, "grad_norm": 0.5986596345901489, "learning_rate": 7.212013148295333e-05, "loss": 0.01201540045440197, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.01209, "step": 209, "tokens/total": 6774872, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 95517} +{"epoch": 0.8203125, "grad_norm": 0.3132984936237335, "learning_rate": 7.185131535190975e-05, "loss": 0.0009930900996550918, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00099, "step": 210, "tokens/total": 6805816, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 96026} +{"epoch": 0.82421875, "grad_norm": 0.3526618778705597, "learning_rate": 7.158179796885005e-05, "loss": 0.003956771921366453, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00396, "step": 211, "tokens/total": 6837592, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 96477} +{"epoch": 0.828125, "grad_norm": 0.5521056056022644, "learning_rate": 7.131159054949273e-05, "loss": 0.003333116415888071, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00334, "step": 212, "tokens/total": 6874384, "tokens/train_per_sec_per_gpu": 6.99, "tokens/trainable": 96951} +{"epoch": 0.83203125, "grad_norm": 0.0089985067024827, "learning_rate": 7.104070433827139e-05, "loss": 8.30259668873623e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00008, "step": 213, "tokens/total": 6905872, "tokens/train_per_sec_per_gpu": 7.04, "tokens/trainable": 97350} +{"epoch": 0.8359375, "grad_norm": 0.3997233211994171, "learning_rate": 7.076915060786705e-05, "loss": 0.0069969394244253635, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.00702, "step": 214, "tokens/total": 6937984, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 97792} +{"epoch": 0.83984375, "grad_norm": 0.1313042938709259, "learning_rate": 7.049694065873882e-05, "loss": 0.00073945929761976, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00074, "step": 215, "tokens/total": 6970704, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 98257} +{"epoch": 0.84375, "grad_norm": 0.39168021082878113, "learning_rate": 7.022408581865382e-05, "loss": 0.004851666744798422, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00486, "step": 216, "tokens/total": 7004616, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 98731} +{"epoch": 0.84765625, "grad_norm": 0.17894238233566284, "learning_rate": 6.99505974422157e-05, "loss": 0.001485921791754663, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00149, "step": 217, "tokens/total": 7036880, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 99212} +{"epoch": 0.8515625, "grad_norm": 0.1829981654882431, "learning_rate": 6.967648691039213e-05, "loss": 0.0010607837466523051, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00106, "step": 218, "tokens/total": 7068208, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 99654} +{"epoch": 0.85546875, "grad_norm": 0.1848425716161728, "learning_rate": 6.940176563004123e-05, "loss": 0.000798188557382673, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.0008, "step": 219, "tokens/total": 7098112, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 100086} +{"epoch": 0.859375, "grad_norm": 0.30629077553749084, "learning_rate": 6.912644503343682e-05, "loss": 0.006870434153825045, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00689, "step": 220, "tokens/total": 7132432, "tokens/train_per_sec_per_gpu": 7.09, "tokens/trainable": 100524} +{"epoch": 0.86328125, "grad_norm": 0.08995433896780014, "learning_rate": 6.885053657779273e-05, "loss": 0.0009326079743914306, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.00093, "step": 221, "tokens/total": 7164016, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 100996} +{"epoch": 0.8671875, "grad_norm": 0.026163766160607338, "learning_rate": 6.857405174478604e-05, "loss": 0.00022770684154238552, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00023, "step": 222, "tokens/total": 7198344, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 101449} +{"epoch": 0.87109375, "grad_norm": 0.21332208812236786, "learning_rate": 6.82970020400792e-05, "loss": 0.0022964845411479473, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.0023, "step": 223, "tokens/total": 7232192, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 101905} +{"epoch": 0.875, "grad_norm": 0.46172356605529785, "learning_rate": 6.801939899284132e-05, "loss": 0.00538953160867095, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.0054, "step": 224, "tokens/total": 7266984, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 102411} +{"epoch": 0.87890625, "grad_norm": 0.032920412719249725, "learning_rate": 6.774125415526827e-05, "loss": 0.00029695304692722857, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.0003, "step": 225, "tokens/total": 7299512, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 102882} +{"epoch": 0.8828125, "grad_norm": 0.010127835907042027, "learning_rate": 6.746257910210214e-05, "loss": 4.7338260628748685e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00005, "step": 226, "tokens/total": 7331120, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 103332} +{"epoch": 0.88671875, "grad_norm": 0.11157752573490143, "learning_rate": 6.718338543014937e-05, "loss": 0.0010420706821605563, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00104, "step": 227, "tokens/total": 7360504, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 103790} +{"epoch": 0.890625, "grad_norm": 0.19154132902622223, "learning_rate": 6.69036847577983e-05, "loss": 0.002450190018862486, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00245, "step": 228, "tokens/total": 7390760, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 104246} +{"epoch": 0.89453125, "grad_norm": 0.10422692447900772, "learning_rate": 6.662348872453553e-05, "loss": 0.0012624174123629928, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00126, "step": 229, "tokens/total": 7423352, "tokens/train_per_sec_per_gpu": 10.3, "tokens/trainable": 104707} +{"epoch": 0.8984375, "grad_norm": 0.5834015607833862, "learning_rate": 6.63428089904618e-05, "loss": 0.014032207429409027, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.01413, "step": 230, "tokens/total": 7457152, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 105167} +{"epoch": 0.90234375, "grad_norm": 0.14721624553203583, "learning_rate": 6.60616572358065e-05, "loss": 0.0007897901232354343, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00079, "step": 231, "tokens/total": 7488648, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 105576} +{"epoch": 0.90625, "grad_norm": 0.9458854794502258, "learning_rate": 6.578004516044172e-05, "loss": 0.009252461604773998, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.0093, "step": 232, "tokens/total": 7522160, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 106043} +{"epoch": 0.91015625, "grad_norm": 0.7050340175628662, "learning_rate": 6.549798448339548e-05, "loss": 0.010553729720413685, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.01061, "step": 233, "tokens/total": 7557336, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 106506} +{"epoch": 0.9140625, "grad_norm": 0.1300826072692871, "learning_rate": 6.521548694236384e-05, "loss": 0.002399969380348921, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.0024, "step": 234, "tokens/total": 7590432, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 106951} +{"epoch": 0.91796875, "grad_norm": 0.5703305006027222, "learning_rate": 6.493256429322259e-05, "loss": 0.020137228071689606, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.02034, "step": 235, "tokens/total": 7623240, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 107382} +{"epoch": 0.921875, "grad_norm": 1.0286340713500977, "learning_rate": 6.464922830953799e-05, "loss": 0.00630530109629035, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00633, "step": 236, "tokens/total": 7657968, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 107814} +{"epoch": 0.92578125, "grad_norm": 0.3597513437271118, "learning_rate": 6.436549078207688e-05, "loss": 0.007731202524155378, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00776, "step": 237, "tokens/total": 7689320, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 108274} +{"epoch": 0.9296875, "grad_norm": 0.22407767176628113, "learning_rate": 6.408136351831592e-05, "loss": 0.004541541915386915, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00455, "step": 238, "tokens/total": 7721568, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 108714} +{"epoch": 0.93359375, "grad_norm": 0.05325715243816376, "learning_rate": 6.379685834195036e-05, "loss": 0.0008053740602917969, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00081, "step": 239, "tokens/total": 7756464, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 109220} +{"epoch": 0.9375, "grad_norm": 0.1454145759344101, "learning_rate": 6.351198709240186e-05, "loss": 0.002379673533141613, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00238, "step": 240, "tokens/total": 7791616, "tokens/train_per_sec_per_gpu": 6.89, "tokens/trainable": 109672} +{"epoch": 0.94140625, "grad_norm": 0.2588263154029846, "learning_rate": 6.32267616243259e-05, "loss": 0.003231235546991229, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00324, "step": 241, "tokens/total": 7823520, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 110135} +{"epoch": 0.9453125, "grad_norm": 0.19249819219112396, "learning_rate": 6.294119380711849e-05, "loss": 0.0030140362214297056, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00302, "step": 242, "tokens/total": 7854424, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 110563} +{"epoch": 0.94921875, "grad_norm": 1.1727542877197266, "learning_rate": 6.265529552442209e-05, "loss": 0.031229788437485695, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.03172, "step": 243, "tokens/total": 7887160, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 111049} +{"epoch": 0.953125, "grad_norm": 0.1491570919752121, "learning_rate": 6.236907867363127e-05, "loss": 0.001559332013130188, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00156, "step": 244, "tokens/total": 7918736, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 111495} +{"epoch": 0.95703125, "grad_norm": 0.7065733075141907, "learning_rate": 6.208255516539749e-05, "loss": 0.009991934522986412, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.01004, "step": 245, "tokens/total": 7949448, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 111968} +{"epoch": 0.9609375, "grad_norm": 0.18821805715560913, "learning_rate": 6.179573692313344e-05, "loss": 0.0014056582003831863, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00141, "step": 246, "tokens/total": 7984680, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 112416} +{"epoch": 0.96484375, "grad_norm": 0.3272077739238739, "learning_rate": 6.150863588251694e-05, "loss": 0.006363274995237589, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00638, "step": 247, "tokens/total": 8019816, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 112882} +{"epoch": 0.96875, "grad_norm": 0.40005576610565186, "learning_rate": 6.122126399099419e-05, "loss": 0.0037099954206496477, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00372, "step": 248, "tokens/total": 8056272, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 113390} +{"epoch": 0.97265625, "grad_norm": 0.5985520482063293, "learning_rate": 6.0933633207282615e-05, "loss": 0.011078556068241596, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.01114, "step": 249, "tokens/total": 8090224, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 113904} +{"epoch": 0.9765625, "grad_norm": 0.04072821885347366, "learning_rate": 6.064575550087316e-05, "loss": 0.0008203086908906698, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00082, "step": 250, "tokens/total": 8124528, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 114351} +{"epoch": 0.98046875, "grad_norm": 2.4497697353363037, "learning_rate": 6.0357642851532245e-05, "loss": 0.002752883592620492, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00276, "step": 251, "tokens/total": 8159688, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 114842} +{"epoch": 0.984375, "grad_norm": 0.2751323878765106, "learning_rate": 6.0069307248803294e-05, "loss": 0.003993778023868799, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.004, "step": 252, "tokens/total": 8191800, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 115263} +{"epoch": 0.98828125, "grad_norm": 0.127595916390419, "learning_rate": 5.9780760691507635e-05, "loss": 0.0011966133024543524, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.0012, "step": 253, "tokens/total": 8221752, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 115703} +{"epoch": 0.9921875, "grad_norm": 0.3066467344760895, "learning_rate": 5.9492015187245334e-05, "loss": 0.0030060862191021442, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00301, "step": 254, "tokens/total": 8255648, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 116157} +{"epoch": 0.99609375, "grad_norm": 0.18706175684928894, "learning_rate": 5.920308275189541e-05, "loss": 0.003232149872928858, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00324, "step": 255, "tokens/total": 8286664, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 116567} +{"epoch": 1.0, "grad_norm": 0.08495759218931198, "learning_rate": 5.8913975409115874e-05, "loss": 0.001980561064556241, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00198, "step": 256, "tokens/total": 8318368, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 117030} +{"epoch": 1.00390625, "grad_norm": 0.029063375666737556, "learning_rate": 5.8624705189843395e-05, "loss": 0.0006222944357432425, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00062, "step": 257, "tokens/total": 8353336, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 117517} +{"epoch": 1.0078125, "grad_norm": 0.09490996599197388, "learning_rate": 5.833528413179249e-05, "loss": 0.0013566571287810802, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00136, "step": 258, "tokens/total": 8389112, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 118015} +{"epoch": 1.01171875, "grad_norm": 0.4713039994239807, "learning_rate": 5.80457242789548e-05, "loss": 0.004946345463395119, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00496, "step": 259, "tokens/total": 8426000, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 118520} +{"epoch": 1.015625, "grad_norm": 0.20682132244110107, "learning_rate": 5.77560376810977e-05, "loss": 0.0012283121468499303, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.00123, "step": 260, "tokens/total": 8458568, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 118992} +{"epoch": 1.01953125, "grad_norm": 0.014483354985713959, "learning_rate": 5.7466236393263005e-05, "loss": 0.0001910147548187524, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00019, "step": 261, "tokens/total": 8490840, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 119438} +{"epoch": 1.0234375, "grad_norm": 0.10861042141914368, "learning_rate": 5.717633247526522e-05, "loss": 0.0017713115084916353, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00177, "step": 262, "tokens/total": 8521792, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 119881} +{"epoch": 1.02734375, "grad_norm": 0.090813547372818, "learning_rate": 5.688633799118971e-05, "loss": 0.0011454196646809578, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00115, "step": 263, "tokens/total": 8551336, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 120285} +{"epoch": 1.03125, "grad_norm": 0.010683647356927395, "learning_rate": 5.659626500889066e-05, "loss": 0.00020940742979291826, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00021, "step": 264, "tokens/total": 8579872, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 120755} +{"epoch": 1.03515625, "grad_norm": 0.0686308965086937, "learning_rate": 5.6306125599488905e-05, "loss": 0.001163370325230062, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00116, "step": 265, "tokens/total": 8608920, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 121194} +{"epoch": 1.0390625, "grad_norm": 0.11580066382884979, "learning_rate": 5.601593183686955e-05, "loss": 0.0012373412027955055, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00124, "step": 266, "tokens/total": 8643600, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 121670} +{"epoch": 1.04296875, "grad_norm": 0.19228462874889374, "learning_rate": 5.572569579717961e-05, "loss": 0.0030203992500901222, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00302, "step": 267, "tokens/total": 8676032, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 122142} +{"epoch": 1.046875, "grad_norm": 0.019759345799684525, "learning_rate": 5.543542955832538e-05, "loss": 0.00015812092169653624, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00016, "step": 268, "tokens/total": 8713240, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 122585} +{"epoch": 1.05078125, "grad_norm": 0.06483939290046692, "learning_rate": 5.514514519946986e-05, "loss": 0.0007411661790683866, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00074, "step": 269, "tokens/total": 8751664, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 123091} +{"epoch": 1.0546875, "grad_norm": 0.037159860134124756, "learning_rate": 5.485485480053015e-05, "loss": 0.0002460320829413831, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00025, "step": 270, "tokens/total": 8786112, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 123505} +{"epoch": 1.05859375, "grad_norm": 0.03332080319523811, "learning_rate": 5.4564570441674645e-05, "loss": 0.0002139806456398219, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00021, "step": 271, "tokens/total": 8816920, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 123953} +{"epoch": 1.0625, "grad_norm": 0.08389882743358612, "learning_rate": 5.42743042028204e-05, "loss": 0.00039578875293955207, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.0004, "step": 272, "tokens/total": 8851408, "tokens/train_per_sec_per_gpu": 6.69, "tokens/trainable": 124400} +{"epoch": 1.06640625, "grad_norm": 0.18473444879055023, "learning_rate": 5.3984068163130464e-05, "loss": 0.0015757185174152255, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00158, "step": 273, "tokens/total": 8883768, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 124870} +{"epoch": 1.0703125, "grad_norm": 0.6748048067092896, "learning_rate": 5.369387440051111e-05, "loss": 0.006585883442312479, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00661, "step": 274, "tokens/total": 8916200, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 125333} +{"epoch": 1.07421875, "grad_norm": 0.2899494469165802, "learning_rate": 5.340373499110935e-05, "loss": 0.00033736106706783175, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00034, "step": 275, "tokens/total": 8949920, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 125834} +{"epoch": 1.078125, "grad_norm": 0.0011436828644946218, "learning_rate": 5.3113662008810304e-05, "loss": 1.9166931451763958e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00002, "step": 276, "tokens/total": 8986448, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 126316} +{"epoch": 1.08203125, "grad_norm": 0.0271908026188612, "learning_rate": 5.282366752473479e-05, "loss": 0.00016475819575134665, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00016, "step": 277, "tokens/total": 9020600, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 126798} +{"epoch": 1.0859375, "grad_norm": 0.06780292838811874, "learning_rate": 5.2533763606737005e-05, "loss": 0.0008010282181203365, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.0008, "step": 278, "tokens/total": 9056040, "tokens/train_per_sec_per_gpu": 6.8, "tokens/trainable": 127223} +{"epoch": 1.08984375, "grad_norm": 0.006906953640282154, "learning_rate": 5.224396231890232e-05, "loss": 7.677912071812898e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00008, "step": 279, "tokens/total": 9087072, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 127672} +{"epoch": 1.09375, "grad_norm": 0.40013614296913147, "learning_rate": 5.195427572104522e-05, "loss": 0.00960432831197977, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00965, "step": 280, "tokens/total": 9118376, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 128116} +{"epoch": 1.09765625, "grad_norm": 0.48583588004112244, "learning_rate": 5.166471586820751e-05, "loss": 0.005440729670226574, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00546, "step": 281, "tokens/total": 9152704, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 128589} +{"epoch": 1.1015625, "grad_norm": 0.07756321132183075, "learning_rate": 5.1375294810156615e-05, "loss": 0.0004340382874943316, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00043, "step": 282, "tokens/total": 9184448, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 129036} +{"epoch": 1.10546875, "grad_norm": 0.2191617637872696, "learning_rate": 5.1086024590884144e-05, "loss": 0.0016874114517122507, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00169, "step": 283, "tokens/total": 9217392, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 129485} +{"epoch": 1.109375, "grad_norm": 0.4568343460559845, "learning_rate": 5.079691724810461e-05, "loss": 0.004167763516306877, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00418, "step": 284, "tokens/total": 9252016, "tokens/train_per_sec_per_gpu": 6.98, "tokens/trainable": 129920} +{"epoch": 1.11328125, "grad_norm": 0.0030521622393280268, "learning_rate": 5.0507984812754684e-05, "loss": 3.216528057237156e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00003, "step": 285, "tokens/total": 9284008, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 130417} +{"epoch": 1.1171875, "grad_norm": 0.4599098265171051, "learning_rate": 5.021923930849237e-05, "loss": 0.005820862948894501, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00584, "step": 286, "tokens/total": 9315984, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 130903} +{"epoch": 1.12109375, "grad_norm": 0.10079014301300049, "learning_rate": 4.99306927511967e-05, "loss": 0.00020607020996976644, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00021, "step": 287, "tokens/total": 9347376, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 131343} +{"epoch": 1.125, "grad_norm": 0.07628873735666275, "learning_rate": 4.964235714846775e-05, "loss": 0.00044522318057715893, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00045, "step": 288, "tokens/total": 9377424, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 131763} +{"epoch": 1.12890625, "grad_norm": 0.06326106190681458, "learning_rate": 4.9354244499126866e-05, "loss": 0.0002474038046784699, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00025, "step": 289, "tokens/total": 9408104, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 132229} +{"epoch": 1.1328125, "grad_norm": 0.0782083123922348, "learning_rate": 4.90663667927174e-05, "loss": 0.0007982982788234949, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.0008, "step": 290, "tokens/total": 9439944, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 132678} +{"epoch": 1.13671875, "grad_norm": 0.25077375769615173, "learning_rate": 4.877873600900581e-05, "loss": 0.0021987587679177523, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.0022, "step": 291, "tokens/total": 9472104, "tokens/train_per_sec_per_gpu": 6.1, "tokens/trainable": 133136} +{"epoch": 1.140625, "grad_norm": 0.002207247307524085, "learning_rate": 4.849136411748306e-05, "loss": 2.1837207896169275e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00002, "step": 292, "tokens/total": 9508336, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 133608} +{"epoch": 1.14453125, "grad_norm": 0.11637427657842636, "learning_rate": 4.8204263076866574e-05, "loss": 0.0010148888686671853, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00102, "step": 293, "tokens/total": 9541552, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 134108} +{"epoch": 1.1484375, "grad_norm": 0.0034992422442883253, "learning_rate": 4.791744483460251e-05, "loss": 1.712257471808698e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00002, "step": 294, "tokens/total": 9575072, "tokens/train_per_sec_per_gpu": 6.48, "tokens/trainable": 134541} +{"epoch": 1.15234375, "grad_norm": 0.003327498445287347, "learning_rate": 4.7630921326368736e-05, "loss": 2.8287873647059314e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00003, "step": 295, "tokens/total": 9610344, "tokens/train_per_sec_per_gpu": 6.31, "tokens/trainable": 134966} +{"epoch": 1.15625, "grad_norm": 0.14677289128303528, "learning_rate": 4.7344704475577916e-05, "loss": 0.001395452651195228, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.0014, "step": 296, "tokens/total": 9639728, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 135402} +{"epoch": 1.16015625, "grad_norm": 0.33610621094703674, "learning_rate": 4.705880619288153e-05, "loss": 0.004676234908401966, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00469, "step": 297, "tokens/total": 9671424, "tokens/train_per_sec_per_gpu": 6.98, "tokens/trainable": 135804} +{"epoch": 1.1640625, "grad_norm": 0.07633072137832642, "learning_rate": 4.677323837567412e-05, "loss": 0.0008857526117935777, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00089, "step": 298, "tokens/total": 9700808, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 136231} +{"epoch": 1.16796875, "grad_norm": 0.03587159886956215, "learning_rate": 4.6488012907598146e-05, "loss": 0.0003368086472619325, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00034, "step": 299, "tokens/total": 9732312, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 136705} +{"epoch": 1.171875, "grad_norm": 0.021329516544938087, "learning_rate": 4.620314165804964e-05, "loss": 0.00014229619409888983, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00014, "step": 300, "tokens/total": 9764624, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 137178} +{"epoch": 1.17578125, "grad_norm": 0.07608292251825333, "learning_rate": 4.591863648168407e-05, "loss": 0.00022333300148602575, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00022, "step": 301, "tokens/total": 9798368, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 137643} +{"epoch": 1.1796875, "grad_norm": 0.39978787302970886, "learning_rate": 4.5634509217923135e-05, "loss": 0.00311135221272707, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00312, "step": 302, "tokens/total": 9830192, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 138078} +{"epoch": 1.18359375, "grad_norm": 0.009449074044823647, "learning_rate": 4.535077169046201e-05, "loss": 7.919113704701886e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00008, "step": 303, "tokens/total": 9863112, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 138515} +{"epoch": 1.1875, "grad_norm": 1.0652533769607544, "learning_rate": 4.506743570677743e-05, "loss": 0.005658562295138836, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00567, "step": 304, "tokens/total": 9893192, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 138948} +{"epoch": 1.19140625, "grad_norm": 0.5878549814224243, "learning_rate": 4.478451305763618e-05, "loss": 0.0038579886313527822, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00387, "step": 305, "tokens/total": 9925712, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 139408} +{"epoch": 1.1953125, "grad_norm": 0.00535372830927372, "learning_rate": 4.450201551660454e-05, "loss": 2.9943843401269987e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00003, "step": 306, "tokens/total": 9955736, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 139840} +{"epoch": 1.19921875, "grad_norm": 0.22799859941005707, "learning_rate": 4.4219954839558276e-05, "loss": 0.0015370867913588881, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00154, "step": 307, "tokens/total": 9987160, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 140281} +{"epoch": 1.203125, "grad_norm": 0.4589921534061432, "learning_rate": 4.393834276419352e-05, "loss": 0.0012810053303837776, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00128, "step": 308, "tokens/total": 10017360, "tokens/train_per_sec_per_gpu": 10.94, "tokens/trainable": 140776} +{"epoch": 1.20703125, "grad_norm": 0.00289175845682621, "learning_rate": 4.36571910095382e-05, "loss": 1.6600173694314435e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00002, "step": 309, "tokens/total": 10052056, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 141228} +{"epoch": 1.2109375, "grad_norm": 0.044091928750276566, "learning_rate": 4.337651127546448e-05, "loss": 0.0002900003455579281, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00029, "step": 310, "tokens/total": 10085424, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 141679} +{"epoch": 1.21484375, "grad_norm": 0.6197790503501892, "learning_rate": 4.3096315242201736e-05, "loss": 0.0029841861687600613, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00299, "step": 311, "tokens/total": 10116824, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 142122} +{"epoch": 1.21875, "grad_norm": 0.005352986976504326, "learning_rate": 4.2816614569850635e-05, "loss": 4.980878293281421e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00005, "step": 312, "tokens/total": 10150376, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 142561} +{"epoch": 1.22265625, "grad_norm": 0.3998185396194458, "learning_rate": 4.2537420897897864e-05, "loss": 0.004244031384587288, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00425, "step": 313, "tokens/total": 10184432, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 143046} +{"epoch": 1.2265625, "grad_norm": 0.021129906177520752, "learning_rate": 4.225874584473174e-05, "loss": 0.00016016581503208727, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00016, "step": 314, "tokens/total": 10214848, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 143493} +{"epoch": 1.23046875, "grad_norm": 0.31497758626937866, "learning_rate": 4.19806010071587e-05, "loss": 0.002916214754804969, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00292, "step": 315, "tokens/total": 10247952, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 143956} +{"epoch": 1.234375, "grad_norm": 0.5200443863868713, "learning_rate": 4.170299795992081e-05, "loss": 0.003093466628342867, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.0031, "step": 316, "tokens/total": 10280648, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 144411} +{"epoch": 1.23828125, "grad_norm": 0.05680787190794945, "learning_rate": 4.142594825521398e-05, "loss": 0.0004991499008610845, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.0005, "step": 317, "tokens/total": 10317712, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 144892} +{"epoch": 1.2421875, "grad_norm": 0.19338616728782654, "learning_rate": 4.114946342220728e-05, "loss": 0.001407327363267541, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00141, "step": 318, "tokens/total": 10351632, "tokens/train_per_sec_per_gpu": 6.86, "tokens/trainable": 145339} +{"epoch": 1.24609375, "grad_norm": 0.09533965587615967, "learning_rate": 4.087355496656321e-05, "loss": 0.0005868331645615399, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00059, "step": 319, "tokens/total": 10383336, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 145811} +{"epoch": 1.25, "grad_norm": 0.2528733015060425, "learning_rate": 4.05982343699588e-05, "loss": 0.0018387367017567158, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00184, "step": 320, "tokens/total": 10418016, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 146314} +{"epoch": 1.25390625, "grad_norm": 0.014641443267464638, "learning_rate": 4.0323513089607876e-05, "loss": 0.00024221833155024797, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00024, "step": 321, "tokens/total": 10451600, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 146781} +{"epoch": 1.2578125, "grad_norm": 0.5515584945678711, "learning_rate": 4.004940255778431e-05, "loss": 0.004982829559594393, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.005, "step": 322, "tokens/total": 10487568, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 147225} +{"epoch": 1.26171875, "grad_norm": 0.31141579151153564, "learning_rate": 3.977591418134619e-05, "loss": 0.001479750731959939, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00148, "step": 323, "tokens/total": 10520976, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 147673} +{"epoch": 1.265625, "grad_norm": 0.02428414300084114, "learning_rate": 3.95030593412612e-05, "loss": 0.00014602199371438473, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00015, "step": 324, "tokens/total": 10552776, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 148103} +{"epoch": 1.26953125, "grad_norm": 0.0054364693351089954, "learning_rate": 3.923084939213296e-05, "loss": 3.884535180986859e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00004, "step": 325, "tokens/total": 10582576, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 148535} +{"epoch": 1.2734375, "grad_norm": 0.006446531508117914, "learning_rate": 3.895929566172861e-05, "loss": 7.54538195906207e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00008, "step": 326, "tokens/total": 10615600, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 148999} +{"epoch": 1.27734375, "grad_norm": 0.012755551375448704, "learning_rate": 3.868840945050728e-05, "loss": 0.00014152801304589957, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00014, "step": 327, "tokens/total": 10650688, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 149431} +{"epoch": 1.28125, "grad_norm": 0.04040026664733887, "learning_rate": 3.841820203114995e-05, "loss": 0.00019852227705996484, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.0002, "step": 328, "tokens/total": 10681816, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 149886} +{"epoch": 1.28515625, "grad_norm": 0.027661064639687538, "learning_rate": 3.814868464809027e-05, "loss": 0.00019163780962117016, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00019, "step": 329, "tokens/total": 10711728, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 150288} +{"epoch": 1.2890625, "grad_norm": 0.013727711513638496, "learning_rate": 3.787986851704667e-05, "loss": 9.908255015034229e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.0001, "step": 330, "tokens/total": 10744952, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 150733} +{"epoch": 1.29296875, "grad_norm": 0.08582206815481186, "learning_rate": 3.7611764824555654e-05, "loss": 0.00035483535612002015, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00035, "step": 331, "tokens/total": 10776192, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 151207} +{"epoch": 1.296875, "grad_norm": 0.04258440434932709, "learning_rate": 3.734438472750619e-05, "loss": 0.0001856097369454801, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00019, "step": 332, "tokens/total": 10813336, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 151694} +{"epoch": 1.30078125, "grad_norm": 0.07674410939216614, "learning_rate": 3.707773935267552e-05, "loss": 0.00020300546020735055, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.0002, "step": 333, "tokens/total": 10847808, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 152188} +{"epoch": 1.3046875, "grad_norm": 0.03355114907026291, "learning_rate": 3.68118397962661e-05, "loss": 0.00020136621606070548, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.0002, "step": 334, "tokens/total": 10879896, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 152596} +{"epoch": 1.30859375, "grad_norm": 0.008446763269603252, "learning_rate": 3.654669712344384e-05, "loss": 1.555111157358624e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00002, "step": 335, "tokens/total": 10908976, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 153052} +{"epoch": 1.3125, "grad_norm": 0.01424316130578518, "learning_rate": 3.628232236787763e-05, "loss": 7.034727605059743e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00007, "step": 336, "tokens/total": 10943152, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 153491} +{"epoch": 1.31640625, "grad_norm": 0.020494654774665833, "learning_rate": 3.6018726531280144e-05, "loss": 6.162349745864049e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00006, "step": 337, "tokens/total": 10978184, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 153983} +{"epoch": 1.3203125, "grad_norm": 0.0009879523422569036, "learning_rate": 3.575592058295017e-05, "loss": 1.534906550659798e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00002, "step": 338, "tokens/total": 11010056, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 154464} +{"epoch": 1.32421875, "grad_norm": 0.0005914591019973159, "learning_rate": 3.549391545931585e-05, "loss": 7.18180535841384e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00001, "step": 339, "tokens/total": 11040968, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 154924} +{"epoch": 1.328125, "grad_norm": 0.001992823788896203, "learning_rate": 3.5232722063479914e-05, "loss": 1.704506757960189e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00002, "step": 340, "tokens/total": 11073424, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 155373} +{"epoch": 1.33203125, "grad_norm": 0.03158099576830864, "learning_rate": 3.49723512647657e-05, "loss": 0.00016675496590323746, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00017, "step": 341, "tokens/total": 11109224, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 155873} +{"epoch": 1.3359375, "grad_norm": 0.01441852655261755, "learning_rate": 3.471281389826491e-05, "loss": 0.00013125305122230202, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00013, "step": 342, "tokens/total": 11139856, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 156278} +{"epoch": 1.33984375, "grad_norm": 0.00825893972069025, "learning_rate": 3.4454120764386764e-05, "loss": 5.476181468111463e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00005, "step": 343, "tokens/total": 11175224, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 156733} +{"epoch": 1.34375, "grad_norm": 0.013495661318302155, "learning_rate": 3.4196282628408526e-05, "loss": 7.465782982762903e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00007, "step": 344, "tokens/total": 11207112, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 157203} +{"epoch": 1.34765625, "grad_norm": 0.043549422174692154, "learning_rate": 3.3939310220027456e-05, "loss": 0.000336467259330675, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00034, "step": 345, "tokens/total": 11240432, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 157707} +{"epoch": 1.3515625, "grad_norm": 0.5257354378700256, "learning_rate": 3.3683214232914404e-05, "loss": 0.0050714327953755856, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00508, "step": 346, "tokens/total": 11274584, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 158180} +{"epoch": 1.35546875, "grad_norm": 0.05549201741814613, "learning_rate": 3.342800532426873e-05, "loss": 0.0005848351866006851, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00059, "step": 347, "tokens/total": 11307376, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 158646} +{"epoch": 1.359375, "grad_norm": 0.03127625957131386, "learning_rate": 3.317369411437484e-05, "loss": 0.0001158618979388848, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00012, "step": 348, "tokens/total": 11340640, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 159115} +{"epoch": 1.36328125, "grad_norm": 0.10215296596288681, "learning_rate": 3.292029118616024e-05, "loss": 0.0005614576512016356, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00056, "step": 349, "tokens/total": 11371632, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 159538} +{"epoch": 1.3671875, "grad_norm": 0.1458968073129654, "learning_rate": 3.266780708475511e-05, "loss": 0.0005348472623154521, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00053, "step": 350, "tokens/total": 11404248, "tokens/train_per_sec_per_gpu": 7.24, "tokens/trainable": 160016} +{"epoch": 1.37109375, "grad_norm": 0.00902843102812767, "learning_rate": 3.241625231705354e-05, "loss": 1.965872070286423e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00002, "step": 351, "tokens/total": 11439552, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 160475} +{"epoch": 1.375, "grad_norm": 0.3980781137943268, "learning_rate": 3.216563735127618e-05, "loss": 0.0005502707790583372, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00055, "step": 352, "tokens/total": 11470944, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 160952} +{"epoch": 1.37890625, "grad_norm": 0.07385551184415817, "learning_rate": 3.191597261653475e-05, "loss": 0.0001873177825473249, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00019, "step": 353, "tokens/total": 11506304, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 161420} +{"epoch": 1.3828125, "grad_norm": 0.42004725337028503, "learning_rate": 3.166726850239794e-05, "loss": 0.00384300434961915, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00385, "step": 354, "tokens/total": 11535424, "tokens/train_per_sec_per_gpu": 10.78, "tokens/trainable": 161868} +{"epoch": 1.38671875, "grad_norm": 0.00551620963960886, "learning_rate": 3.141953535845912e-05, "loss": 4.330368756200187e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00004, "step": 355, "tokens/total": 11565176, "tokens/train_per_sec_per_gpu": 10.07, "tokens/trainable": 162318} +{"epoch": 1.390625, "grad_norm": 0.01040590088814497, "learning_rate": 3.11727834939056e-05, "loss": 3.9192356780404225e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00004, "step": 356, "tokens/total": 11595528, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 162789} +{"epoch": 1.39453125, "grad_norm": 0.572396457195282, "learning_rate": 3.092702317708967e-05, "loss": 0.01298946887254715, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.01307, "step": 357, "tokens/total": 11627248, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 163254} +{"epoch": 1.3984375, "grad_norm": 0.0013009083922952414, "learning_rate": 3.0682264635101276e-05, "loss": 9.909551408782136e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00001, "step": 358, "tokens/total": 11659440, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 163715} +{"epoch": 1.40234375, "grad_norm": 0.004232152830809355, "learning_rate": 3.0438518053342407e-05, "loss": 1.5178122339420952e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00002, "step": 359, "tokens/total": 11693288, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 164197} +{"epoch": 1.40625, "grad_norm": 0.07899797707796097, "learning_rate": 3.0195793575103266e-05, "loss": 0.0004209275939501822, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00042, "step": 360, "tokens/total": 11730888, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 164661} +{"epoch": 1.41015625, "grad_norm": 0.171684131026268, "learning_rate": 2.9954101301140146e-05, "loss": 0.001231911825016141, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00123, "step": 361, "tokens/total": 11765656, "tokens/train_per_sec_per_gpu": 6.8, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.015184512361884117, "learning_rate": 2.9713451289255123e-05, "loss": 0.00012737327779177576, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00013, "step": 362, "tokens/total": 11793456, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 165552} +{"epoch": 1.41796875, "grad_norm": 0.0020704721100628376, "learning_rate": 2.9473853553877484e-05, "loss": 2.010515527217649e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00002, "step": 363, "tokens/total": 11826280, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 166026} +{"epoch": 1.421875, "grad_norm": 0.007157416082918644, "learning_rate": 2.9235318065647e-05, "loss": 5.197343853069469e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00005, "step": 364, "tokens/total": 11863552, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 166486} +{"epoch": 1.42578125, "grad_norm": 0.0006134856375865638, "learning_rate": 2.8997854750998964e-05, "loss": 7.032583198451903e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00001, "step": 365, "tokens/total": 11898616, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 166959} +{"epoch": 1.4296875, "grad_norm": 0.1011909767985344, "learning_rate": 2.8761473491751258e-05, "loss": 0.0003999993787147105, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.0004, "step": 366, "tokens/total": 11930728, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 167394} +{"epoch": 1.43359375, "grad_norm": 0.023814614862203598, "learning_rate": 2.8526184124692883e-05, "loss": 2.2725722374161705e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00002, "step": 367, "tokens/total": 11964976, "tokens/train_per_sec_per_gpu": 6.29, "tokens/trainable": 167832} +{"epoch": 1.4375, "grad_norm": 0.005600878037512302, "learning_rate": 2.829199644117484e-05, "loss": 2.5148841814370826e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00003, "step": 368, "tokens/total": 11996696, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 168242} +{"epoch": 1.44140625, "grad_norm": 0.12386581301689148, "learning_rate": 2.8058920186702553e-05, "loss": 0.0004401668848004192, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00044, "step": 369, "tokens/total": 12033432, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 168715} +{"epoch": 1.4453125, "grad_norm": 0.23638273775577545, "learning_rate": 2.782696506053033e-05, "loss": 0.002528558950871229, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00253, "step": 370, "tokens/total": 12067616, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 169147} +{"epoch": 1.44921875, "grad_norm": 0.0066299415193498135, "learning_rate": 2.7596140715257824e-05, "loss": 2.9609653211082332e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00003, "step": 371, "tokens/total": 12099008, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 169628} +{"epoch": 1.453125, "grad_norm": 0.0017984445439651608, "learning_rate": 2.7366456756428184e-05, "loss": 1.4723393178428523e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00001, "step": 372, "tokens/total": 12131304, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 170085} +{"epoch": 1.45703125, "grad_norm": 0.020236147567629814, "learning_rate": 2.7137922742128486e-05, "loss": 0.0001413742866134271, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00014, "step": 373, "tokens/total": 12164104, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 170584} +{"epoch": 1.4609375, "grad_norm": 0.5225854516029358, "learning_rate": 2.691054818259188e-05, "loss": 0.006186270620673895, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00621, "step": 374, "tokens/total": 12197776, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 171058} +{"epoch": 1.46484375, "grad_norm": 0.15420560538768768, "learning_rate": 2.6684342539801933e-05, "loss": 0.0009652148000895977, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00097, "step": 375, "tokens/total": 12228064, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 171518} +{"epoch": 1.46875, "grad_norm": 0.4066447615623474, "learning_rate": 2.645931522709877e-05, "loss": 0.0034052119590342045, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00341, "step": 376, "tokens/total": 12261688, "tokens/train_per_sec_per_gpu": 10.55, "tokens/trainable": 172007} +{"epoch": 1.47265625, "grad_norm": 0.18306000530719757, "learning_rate": 2.6235475608787365e-05, "loss": 0.0005800652434118092, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00058, "step": 377, "tokens/total": 12294240, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 172468} +{"epoch": 1.4765625, "grad_norm": 0.7760815024375916, "learning_rate": 2.6012832999747916e-05, "loss": 0.010021001100540161, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.01007, "step": 378, "tokens/total": 12324528, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 172923} +{"epoch": 1.48046875, "grad_norm": 0.42357444763183594, "learning_rate": 2.579139666504821e-05, "loss": 0.01811659336090088, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01828, "step": 379, "tokens/total": 12360288, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 173370} +{"epoch": 1.484375, "grad_norm": 0.0026043078396469355, "learning_rate": 2.557117581955798e-05, "loss": 1.1607737178565003e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00001, "step": 380, "tokens/total": 12391216, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 173837} +{"epoch": 1.48828125, "grad_norm": 0.02381674014031887, "learning_rate": 2.5352179627565532e-05, "loss": 9.682677045930177e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.0001, "step": 381, "tokens/total": 12423624, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 174294} +{"epoch": 1.4921875, "grad_norm": 0.0023619283456355333, "learning_rate": 2.5134417202396277e-05, "loss": 1.7670270608505234e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00002, "step": 382, "tokens/total": 12456032, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 174701} +{"epoch": 1.49609375, "grad_norm": 0.006087234243750572, "learning_rate": 2.491789760603361e-05, "loss": 4.04441925638821e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00004, "step": 383, "tokens/total": 12489336, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 175131} +{"epoch": 1.5, "grad_norm": 0.0020945246797055006, "learning_rate": 2.4702629848741764e-05, "loss": 3.62934879376553e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00004, "step": 384, "tokens/total": 12521944, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 175586} +{"epoch": 1.50390625, "grad_norm": 0.4604203701019287, "learning_rate": 2.4488622888690785e-05, "loss": 0.00430371193215251, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00431, "step": 385, "tokens/total": 12554024, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 176026} +{"epoch": 1.5078125, "grad_norm": 1.380599856376648, "learning_rate": 2.427588563158384e-05, "loss": 0.0005171548691578209, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00052, "step": 386, "tokens/total": 12587104, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 176512} +{"epoch": 1.51171875, "grad_norm": 0.018947500735521317, "learning_rate": 2.406442693028651e-05, "loss": 0.0001634508662391454, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00016, "step": 387, "tokens/total": 12617456, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 176943} +{"epoch": 1.515625, "grad_norm": 0.01220768690109253, "learning_rate": 2.3854255584458547e-05, "loss": 9.454677638132125e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00009, "step": 388, "tokens/total": 12652696, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 177370} +{"epoch": 1.51953125, "grad_norm": 0.04627299681305885, "learning_rate": 2.3645380340187508e-05, "loss": 0.000250139826675877, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00025, "step": 389, "tokens/total": 12685792, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 177851} +{"epoch": 1.5234375, "grad_norm": 0.011427781544625759, "learning_rate": 2.3437809889624914e-05, "loss": 0.00011600360448937863, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00012, "step": 390, "tokens/total": 12716120, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 178310} +{"epoch": 1.52734375, "grad_norm": 0.00974294263869524, "learning_rate": 2.3231552870624487e-05, "loss": 0.00013367882638704032, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00013, "step": 391, "tokens/total": 12746216, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 178736} +{"epoch": 1.53125, "grad_norm": 0.12208108603954315, "learning_rate": 2.3026617866382657e-05, "loss": 0.0012075196718797088, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00121, "step": 392, "tokens/total": 12778592, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 179233} +{"epoch": 1.53515625, "grad_norm": 0.1340303122997284, "learning_rate": 2.2823013405081507e-05, "loss": 0.00037773512303829193, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00038, "step": 393, "tokens/total": 12812008, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 179730} +{"epoch": 1.5390625, "grad_norm": 0.004464747849851847, "learning_rate": 2.2620747959533722e-05, "loss": 6.552059494424611e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00007, "step": 394, "tokens/total": 12844264, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 180227} +{"epoch": 1.54296875, "grad_norm": 0.018907252699136734, "learning_rate": 2.2419829946830123e-05, "loss": 0.00021002948051318526, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00021, "step": 395, "tokens/total": 12878328, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 180687} +{"epoch": 1.546875, "grad_norm": 0.12069133669137955, "learning_rate": 2.2220267727989325e-05, "loss": 0.00038716409471817315, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00039, "step": 396, "tokens/total": 12907912, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 181141} +{"epoch": 1.55078125, "grad_norm": 0.20875097811222076, "learning_rate": 2.202206960760984e-05, "loss": 0.004784252494573593, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.0048, "step": 397, "tokens/total": 12941968, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 181648} +{"epoch": 1.5546875, "grad_norm": 0.36716774106025696, "learning_rate": 2.182524383352446e-05, "loss": 0.010294249281287193, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.01035, "step": 398, "tokens/total": 12975240, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 182109} +{"epoch": 1.55859375, "grad_norm": 0.08166062086820602, "learning_rate": 2.1629798596457056e-05, "loss": 0.0004707640619017184, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00047, "step": 399, "tokens/total": 13006912, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 182559} +{"epoch": 1.5625, "grad_norm": 0.06864458322525024, "learning_rate": 2.1435742029681725e-05, "loss": 0.0004941194201819599, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00049, "step": 400, "tokens/total": 13037640, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 182985} +{"epoch": 1.56640625, "grad_norm": 0.03433334454894066, "learning_rate": 2.124308220868431e-05, "loss": 0.0002731874701566994, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00027, "step": 401, "tokens/total": 13070608, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 183440} +{"epoch": 1.5703125, "grad_norm": 0.12079767137765884, "learning_rate": 2.105182715082638e-05, "loss": 0.0009772845078259706, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00098, "step": 402, "tokens/total": 13101784, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 183917} +{"epoch": 1.57421875, "grad_norm": 0.005115099251270294, "learning_rate": 2.0861984815011552e-05, "loss": 6.50161164230667e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00007, "step": 403, "tokens/total": 13133528, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 184348} +{"epoch": 1.578125, "grad_norm": 0.015218960121273994, "learning_rate": 2.0673563101354323e-05, "loss": 0.00016061015776358545, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00016, "step": 404, "tokens/total": 13167088, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 184786} +{"epoch": 1.58203125, "grad_norm": 0.24395491182804108, "learning_rate": 2.0486569850851317e-05, "loss": 0.000869488634634763, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00087, "step": 405, "tokens/total": 13200392, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 185249} +{"epoch": 1.5859375, "grad_norm": 0.18820108473300934, "learning_rate": 2.0301012845054956e-05, "loss": 0.0006331065087579191, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00063, "step": 406, "tokens/total": 13235488, "tokens/train_per_sec_per_gpu": 6.75, "tokens/trainable": 185680} +{"epoch": 1.58984375, "grad_norm": 0.41543373465538025, "learning_rate": 2.011689980574966e-05, "loss": 0.0005822904058732092, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00058, "step": 407, "tokens/total": 13269416, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 186100} +{"epoch": 1.59375, "grad_norm": 0.04585632309317589, "learning_rate": 1.993423839463052e-05, "loss": 0.00041520054219290614, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00042, "step": 408, "tokens/total": 13301360, "tokens/train_per_sec_per_gpu": 10.24, "tokens/trainable": 186565} +{"epoch": 1.59765625, "grad_norm": 0.013204147107899189, "learning_rate": 1.975303621298445e-05, "loss": 0.00019903517386410385, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.0002, "step": 409, "tokens/total": 13336392, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 187017} +{"epoch": 1.6015625, "grad_norm": 0.04801582172513008, "learning_rate": 1.957330080137385e-05, "loss": 0.00018449081107974052, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00018, "step": 410, "tokens/total": 13366072, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 187468} +{"epoch": 1.60546875, "grad_norm": 0.047732677310705185, "learning_rate": 1.9395039639322864e-05, "loss": 0.0005496770027093589, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00055, "step": 411, "tokens/total": 13400832, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 187961} +{"epoch": 1.609375, "grad_norm": 0.008300150744616985, "learning_rate": 1.9218260145006073e-05, "loss": 4.353308759164065e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00004, "step": 412, "tokens/total": 13434920, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 188447} +{"epoch": 1.61328125, "grad_norm": 0.33593446016311646, "learning_rate": 1.904296967493982e-05, "loss": 0.00157443736679852, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00158, "step": 413, "tokens/total": 13466512, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 188903} +{"epoch": 1.6171875, "grad_norm": 0.2148631364107132, "learning_rate": 1.8869175523676064e-05, "loss": 0.002847519004717469, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00285, "step": 414, "tokens/total": 13500352, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 189398} +{"epoch": 1.62109375, "grad_norm": 0.03169994801282883, "learning_rate": 1.869688492349885e-05, "loss": 0.00037600682117044926, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00038, "step": 415, "tokens/total": 13532448, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 189850} +{"epoch": 1.625, "grad_norm": 0.012368932366371155, "learning_rate": 1.85261050441233e-05, "loss": 0.00017666889471001923, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00018, "step": 416, "tokens/total": 13565648, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 190306} +{"epoch": 1.62890625, "grad_norm": 0.2402869611978531, "learning_rate": 1.8356842992397304e-05, "loss": 0.001992004457861185, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00199, "step": 417, "tokens/total": 13597600, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 190746} +{"epoch": 1.6328125, "grad_norm": 0.1397218406200409, "learning_rate": 1.8189105812005714e-05, "loss": 0.0009797491366043687, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00098, "step": 418, "tokens/total": 13630456, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 191156} +{"epoch": 1.63671875, "grad_norm": 0.03388632833957672, "learning_rate": 1.802290048317732e-05, "loss": 0.00020902442338410765, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00021, "step": 419, "tokens/total": 13660896, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 191573} +{"epoch": 1.640625, "grad_norm": 0.010058694519102573, "learning_rate": 1.785823392239424e-05, "loss": 6.259501969907433e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00006, "step": 420, "tokens/total": 13693176, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 192014} +{"epoch": 1.64453125, "grad_norm": 0.060071900486946106, "learning_rate": 1.7695112982104225e-05, "loss": 0.000370395922800526, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00037, "step": 421, "tokens/total": 13725400, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 192453} +{"epoch": 1.6484375, "grad_norm": 0.011024022474884987, "learning_rate": 1.7533544450435433e-05, "loss": 8.672248077346012e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00009, "step": 422, "tokens/total": 13752752, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 192886} +{"epoch": 1.65234375, "grad_norm": 0.00259262858889997, "learning_rate": 1.7373535050913946e-05, "loss": 2.6612171495798975e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00003, "step": 423, "tokens/total": 13786136, "tokens/train_per_sec_per_gpu": 7.04, "tokens/trainable": 193323} +{"epoch": 1.65625, "grad_norm": 0.025876086205244064, "learning_rate": 1.721509144218405e-05, "loss": 0.0003145028604194522, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00031, "step": 424, "tokens/total": 13815904, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 193753} +{"epoch": 1.66015625, "grad_norm": 0.07595101743936539, "learning_rate": 1.705822021773101e-05, "loss": 0.00017186249897349626, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00017, "step": 425, "tokens/total": 13847976, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 194240} +{"epoch": 1.6640625, "grad_norm": 0.00829028058797121, "learning_rate": 1.69029279056068e-05, "loss": 8.87344212969765e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00009, "step": 426, "tokens/total": 13883200, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 194680} +{"epoch": 1.66796875, "grad_norm": 0.03157031908631325, "learning_rate": 1.6749220968158415e-05, "loss": 0.00020241076708771288, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.0002, "step": 427, "tokens/total": 13916080, "tokens/train_per_sec_per_gpu": 10.18, "tokens/trainable": 195186} +{"epoch": 1.671875, "grad_norm": 0.05544786900281906, "learning_rate": 1.659710580175893e-05, "loss": 0.00032055002520792186, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00032, "step": 428, "tokens/total": 13946448, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 195693} +{"epoch": 1.67578125, "grad_norm": 0.029370442032814026, "learning_rate": 1.644658873654133e-05, "loss": 0.00019196954963263124, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00019, "step": 429, "tokens/total": 13975696, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 196130} +{"epoch": 1.6796875, "grad_norm": 0.15232108533382416, "learning_rate": 1.629767603613508e-05, "loss": 0.0005639739683829248, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00056, "step": 430, "tokens/total": 14002616, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 196530} +{"epoch": 1.68359375, "grad_norm": 0.005071453284472227, "learning_rate": 1.615037389740547e-05, "loss": 5.93621407460887e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00006, "step": 431, "tokens/total": 14034848, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 196974} +{"epoch": 1.6875, "grad_norm": 0.016947107389569283, "learning_rate": 1.600468845019576e-05, "loss": 0.00013196690997574478, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00013, "step": 432, "tokens/total": 14069032, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 197454} +{"epoch": 1.69140625, "grad_norm": 0.012851349078118801, "learning_rate": 1.5860625757072092e-05, "loss": 4.9383714213036e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00005, "step": 433, "tokens/total": 14101296, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 197902} +{"epoch": 1.6953125, "grad_norm": 0.027980055660009384, "learning_rate": 1.571819181307116e-05, "loss": 0.00012183563376311213, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00012, "step": 434, "tokens/total": 14135880, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 198374} +{"epoch": 1.69921875, "grad_norm": 0.0210361760109663, "learning_rate": 1.557739254545075e-05, "loss": 0.00015002854343038052, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00015, "step": 435, "tokens/total": 14166040, "tokens/train_per_sec_per_gpu": 10.47, "tokens/trainable": 198846} +{"epoch": 1.703125, "grad_norm": 0.011494336649775505, "learning_rate": 1.543823381344311e-05, "loss": 3.676733103930019e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00004, "step": 436, "tokens/total": 14196032, "tokens/train_per_sec_per_gpu": 10.62, "tokens/trainable": 199320} +{"epoch": 1.70703125, "grad_norm": 0.5373965501785278, "learning_rate": 1.5300721408011114e-05, "loss": 0.0052614761516451836, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00528, "step": 437, "tokens/total": 14228008, "tokens/train_per_sec_per_gpu": 10.05, "tokens/trainable": 199777} +{"epoch": 1.7109375, "grad_norm": 0.021554509177803993, "learning_rate": 1.5164861051607254e-05, "loss": 0.0001344595948467031, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.76, "memory/max_allocated (GiB)": 65.76, "ppl": 1.00013, "step": 438, "tokens/total": 14261744, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 200194} +{"epoch": 1.71484375, "grad_norm": 0.11125767976045609, "learning_rate": 1.5030658397935521e-05, "loss": 0.0001554017508169636, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00016, "step": 439, "tokens/total": 14295208, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 200663} +{"epoch": 1.71875, "grad_norm": 0.01966184563934803, "learning_rate": 1.4898119031716104e-05, "loss": 0.00012599333422258496, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00013, "step": 440, "tokens/total": 14327264, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 201121} +{"epoch": 1.72265625, "grad_norm": 0.001965124858543277, "learning_rate": 1.476724846845306e-05, "loss": 2.4466535251121968e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00002, "step": 441, "tokens/total": 14363088, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 201598} +{"epoch": 1.7265625, "grad_norm": 0.01146615855395794, "learning_rate": 1.463805215420471e-05, "loss": 9.50126996031031e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.0001, "step": 442, "tokens/total": 14396088, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 202068} +{"epoch": 1.73046875, "grad_norm": 0.0005408645374700427, "learning_rate": 1.451053546535705e-05, "loss": 9.029350621858612e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00001, "step": 443, "tokens/total": 14429232, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 202524} +{"epoch": 1.734375, "grad_norm": 0.006567294709384441, "learning_rate": 1.438470370840001e-05, "loss": 7.57182133384049e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00008, "step": 444, "tokens/total": 14463264, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 202996} +{"epoch": 1.73828125, "grad_norm": 0.0026363821234554052, "learning_rate": 1.4260562119706606e-05, "loss": 4.080476355738938e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00004, "step": 445, "tokens/total": 14493552, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 203470} +{"epoch": 1.7421875, "grad_norm": 1.6870391368865967, "learning_rate": 1.413811586531508e-05, "loss": 0.003628334729000926, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00363, "step": 446, "tokens/total": 14525240, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 203919} +{"epoch": 1.74609375, "grad_norm": 0.03146975859999657, "learning_rate": 1.4017370040713884e-05, "loss": 0.00021097563148941845, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00021, "step": 447, "tokens/total": 14559560, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 204376} +{"epoch": 1.75, "grad_norm": 0.060271963477134705, "learning_rate": 1.3898329670629645e-05, "loss": 0.00031021900940686464, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00031, "step": 448, "tokens/total": 14594160, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 204821} +{"epoch": 1.75390625, "grad_norm": 0.007805718574672937, "learning_rate": 1.3780999708818058e-05, "loss": 3.874309186358005e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00004, "step": 449, "tokens/total": 14624224, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 205271} +{"epoch": 1.7578125, "grad_norm": 0.0026328726671636105, "learning_rate": 1.3665385037857758e-05, "loss": 1.3521466826205142e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00001, "step": 450, "tokens/total": 14654984, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 205723} +{"epoch": 1.76171875, "grad_norm": 0.03706131502985954, "learning_rate": 1.3551490468947126e-05, "loss": 0.00035187893081456423, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00035, "step": 451, "tokens/total": 14685440, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 206169} +{"epoch": 1.765625, "grad_norm": 0.01522844284772873, "learning_rate": 1.3439320741704075e-05, "loss": 0.00011710578837664798, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00012, "step": 452, "tokens/total": 14716120, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 206592} +{"epoch": 1.76953125, "grad_norm": 0.00461990712210536, "learning_rate": 1.3328880523968808e-05, "loss": 3.617153561208397e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00004, "step": 453, "tokens/total": 14748424, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 207055} +{"epoch": 1.7734375, "grad_norm": 0.002424706472083926, "learning_rate": 1.3220174411609587e-05, "loss": 3.268092041253112e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00003, "step": 454, "tokens/total": 14781256, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 207494} +{"epoch": 1.77734375, "grad_norm": 0.007246939465403557, "learning_rate": 1.3113206928331471e-05, "loss": 4.2633648263290524e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00004, "step": 455, "tokens/total": 14809552, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 207956} +{"epoch": 1.78125, "grad_norm": 0.0037956577725708485, "learning_rate": 1.300798252548806e-05, "loss": 2.7085865440312773e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00003, "step": 456, "tokens/total": 14841808, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 208408} +{"epoch": 1.78515625, "grad_norm": 0.00359857315197587, "learning_rate": 1.2904505581896265e-05, "loss": 2.0247809516149573e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00002, "step": 457, "tokens/total": 14875896, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 208889} +{"epoch": 1.7890625, "grad_norm": 0.0081919701769948, "learning_rate": 1.2802780403654082e-05, "loss": 8.266109944088385e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00008, "step": 458, "tokens/total": 14908424, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 209336} +{"epoch": 1.79296875, "grad_norm": 0.002387524349614978, "learning_rate": 1.2702811223961408e-05, "loss": 3.17207959596999e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00003, "step": 459, "tokens/total": 14939168, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 209797} +{"epoch": 1.796875, "grad_norm": 0.006075216922909021, "learning_rate": 1.2604602202943861e-05, "loss": 6.792879867134616e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00007, "step": 460, "tokens/total": 14969872, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 210240} +{"epoch": 1.80078125, "grad_norm": 0.02022925391793251, "learning_rate": 1.2508157427479686e-05, "loss": 0.0001822380581870675, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00018, "step": 461, "tokens/total": 15001472, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 210702} +{"epoch": 1.8046875, "grad_norm": 0.16027703881263733, "learning_rate": 1.2413480911029655e-05, "loss": 0.001271837973035872, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00127, "step": 462, "tokens/total": 15030592, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 211140} +{"epoch": 1.80859375, "grad_norm": 0.012603512033820152, "learning_rate": 1.2320576593470082e-05, "loss": 0.00011309823457850143, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00011, "step": 463, "tokens/total": 15063648, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 211575} +{"epoch": 1.8125, "grad_norm": 0.024296177551150322, "learning_rate": 1.2229448340928828e-05, "loss": 0.00016248729662038386, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00016, "step": 464, "tokens/total": 15098248, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 212083} +{"epoch": 1.81640625, "grad_norm": 0.0054872967302799225, "learning_rate": 1.2140099945624458e-05, "loss": 2.5484727302682586e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00003, "step": 465, "tokens/total": 15133216, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 212558} +{"epoch": 1.8203125, "grad_norm": 0.006560523062944412, "learning_rate": 1.205253512570841e-05, "loss": 5.205101479077712e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00005, "step": 466, "tokens/total": 15165416, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 213014} +{"epoch": 1.82421875, "grad_norm": 0.012321488000452518, "learning_rate": 1.1966757525110255e-05, "loss": 0.00010458388715051115, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.0001, "step": 467, "tokens/total": 15193688, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 213465} +{"epoch": 1.828125, "grad_norm": 0.349623441696167, "learning_rate": 1.1882770713386095e-05, "loss": 0.005734116770327091, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00575, "step": 468, "tokens/total": 15228600, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 213940} +{"epoch": 1.83203125, "grad_norm": 0.16640733182430267, "learning_rate": 1.180057818556998e-05, "loss": 0.0013017145683988929, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.0013, "step": 469, "tokens/total": 15262192, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 214415} +{"epoch": 1.8359375, "grad_norm": 0.02127826027572155, "learning_rate": 1.1720183362028494e-05, "loss": 8.007785072550178e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00008, "step": 470, "tokens/total": 15294920, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 214880} +{"epoch": 1.83984375, "grad_norm": 0.669748067855835, "learning_rate": 1.1641589588318387e-05, "loss": 0.0069096265360713005, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00693, "step": 471, "tokens/total": 15327432, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 215334} +{"epoch": 1.84375, "grad_norm": 0.002818204229697585, "learning_rate": 1.1564800135047418e-05, "loss": 2.0312374545028433e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00002, "step": 472, "tokens/total": 15359928, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 215822} +{"epoch": 1.84765625, "grad_norm": 0.04242056608200073, "learning_rate": 1.148981819773816e-05, "loss": 0.00017328561807516962, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00017, "step": 473, "tokens/total": 15391400, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 216239} +{"epoch": 1.8515625, "grad_norm": 0.00038212790968827903, "learning_rate": 1.1416646896695086e-05, "loss": 7.725491741439328e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00001, "step": 474, "tokens/total": 15424144, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 216691} +{"epoch": 1.85546875, "grad_norm": 0.0018142469925805926, "learning_rate": 1.1345289276874717e-05, "loss": 2.2732909201295115e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00002, "step": 475, "tokens/total": 15457680, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 217162} +{"epoch": 1.859375, "grad_norm": 0.01118460949510336, "learning_rate": 1.1275748307758873e-05, "loss": 4.832133708987385e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00005, "step": 476, "tokens/total": 15492056, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 217638} +{"epoch": 1.86328125, "grad_norm": 0.019082140177488327, "learning_rate": 1.1208026883231147e-05, "loss": 0.000133924389956519, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00013, "step": 477, "tokens/total": 15521544, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 218053} +{"epoch": 1.8671875, "grad_norm": 0.0028094477020204067, "learning_rate": 1.1142127821456433e-05, "loss": 3.9136604755185544e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00004, "step": 478, "tokens/total": 15553320, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 218503} +{"epoch": 1.87109375, "grad_norm": 0.061419565230607986, "learning_rate": 1.1078053864763674e-05, "loss": 0.0002244981296826154, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00022, "step": 479, "tokens/total": 15586208, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 218964} +{"epoch": 1.875, "grad_norm": 0.4348451495170593, "learning_rate": 1.1015807679531756e-05, "loss": 0.009149789810180664, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00919, "step": 480, "tokens/total": 15616432, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 219412} +{"epoch": 1.87890625, "grad_norm": 0.14451509714126587, "learning_rate": 1.0955391856078528e-05, "loss": 0.0006265959818847477, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00063, "step": 481, "tokens/total": 15650056, "tokens/train_per_sec_per_gpu": 10.11, "tokens/trainable": 219886} +{"epoch": 1.8828125, "grad_norm": 0.0025766431353986263, "learning_rate": 1.0896808908553007e-05, "loss": 3.2910083973547444e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00003, "step": 482, "tokens/total": 15680336, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 220343} +{"epoch": 1.88671875, "grad_norm": 0.13726790249347687, "learning_rate": 1.0840061274830763e-05, "loss": 0.0009568004170432687, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00096, "step": 483, "tokens/total": 15711552, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 220825} +{"epoch": 1.890625, "grad_norm": 0.002171273808926344, "learning_rate": 1.0785151316412473e-05, "loss": 1.8641458154888824e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00002, "step": 484, "tokens/total": 15744664, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 221282} +{"epoch": 1.89453125, "grad_norm": 0.01616428978741169, "learning_rate": 1.0732081318325639e-05, "loss": 0.00010329447104595602, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.0001, "step": 485, "tokens/total": 15782800, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 221780} +{"epoch": 1.8984375, "grad_norm": 0.010991828516125679, "learning_rate": 1.0680853489029501e-05, "loss": 0.00010300326539436355, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.0001, "step": 486, "tokens/total": 15817368, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 222210} +{"epoch": 1.90234375, "grad_norm": 0.09499260783195496, "learning_rate": 1.0631469960323152e-05, "loss": 0.0009093469125218689, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00091, "step": 487, "tokens/total": 15850408, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 222646} +{"epoch": 1.90625, "grad_norm": 0.09361634403467178, "learning_rate": 1.0583932787256783e-05, "loss": 0.00014455197378993034, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00014, "step": 488, "tokens/total": 15881448, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 223066} +{"epoch": 1.91015625, "grad_norm": 0.009352521039545536, "learning_rate": 1.0538243948046206e-05, "loss": 6.783424760214984e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00007, "step": 489, "tokens/total": 15913736, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 223512} +{"epoch": 1.9140625, "grad_norm": 0.010727930814027786, "learning_rate": 1.0494405343990523e-05, "loss": 6.208494596648961e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00006, "step": 490, "tokens/total": 15944568, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 223956} +{"epoch": 1.91796875, "grad_norm": 0.0022497959434986115, "learning_rate": 1.0452418799392985e-05, "loss": 1.9719363990589045e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00002, "step": 491, "tokens/total": 15976352, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 224416} +{"epoch": 1.921875, "grad_norm": 0.1897134929895401, "learning_rate": 1.0412286061485102e-05, "loss": 0.00106707657687366, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00107, "step": 492, "tokens/total": 16010392, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 224883} +{"epoch": 1.92578125, "grad_norm": 0.0641198381781578, "learning_rate": 1.03740088003539e-05, "loss": 0.00023501603573095053, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00024, "step": 493, "tokens/total": 16039176, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 225324} +{"epoch": 1.9296875, "grad_norm": 0.007822454907000065, "learning_rate": 1.0337588608872463e-05, "loss": 4.4598298700293526e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00004, "step": 494, "tokens/total": 16067928, "tokens/train_per_sec_per_gpu": 10.32, "tokens/trainable": 225783} +{"epoch": 1.93359375, "grad_norm": 1.2150293588638306, "learning_rate": 1.0303027002633622e-05, "loss": 0.010159033350646496, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.01021, "step": 495, "tokens/total": 16100264, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 226281} +{"epoch": 1.9375, "grad_norm": 0.007294784765690565, "learning_rate": 1.0270325419886884e-05, "loss": 3.4181724913651124e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00003, "step": 496, "tokens/total": 16132504, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 226735} +{"epoch": 1.94140625, "grad_norm": 0.0031728751491755247, "learning_rate": 1.0239485221478599e-05, "loss": 3.3301439543720335e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00003, "step": 497, "tokens/total": 16167064, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 227198} +{"epoch": 1.9453125, "grad_norm": 0.0007797962171025574, "learning_rate": 1.0210507690795292e-05, "loss": 5.481091193360044e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00001, "step": 498, "tokens/total": 16198584, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 227623} +{"epoch": 1.94921875, "grad_norm": 0.03156643733382225, "learning_rate": 1.0183394033710305e-05, "loss": 0.00021004377049393952, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00021, "step": 499, "tokens/total": 16229584, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 228074} +{"epoch": 1.953125, "grad_norm": 0.12398778647184372, "learning_rate": 1.0158145378533583e-05, "loss": 0.0008377333870157599, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00084, "step": 500, "tokens/total": 16260232, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 228541} +{"epoch": 1.95703125, "grad_norm": 0.021754834800958633, "learning_rate": 1.0134762775964726e-05, "loss": 3.295419446658343e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00003, "step": 501, "tokens/total": 16292352, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 229031} +{"epoch": 1.9609375, "grad_norm": 0.0399908721446991, "learning_rate": 1.0113247199049278e-05, "loss": 0.0002057485398836434, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00021, "step": 502, "tokens/total": 16325424, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 229512} +{"epoch": 1.96484375, "grad_norm": 0.3822439908981323, "learning_rate": 1.0093599543138205e-05, "loss": 0.00199120189063251, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00199, "step": 503, "tokens/total": 16359184, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 229989} +{"epoch": 1.96875, "grad_norm": 0.0975574180483818, "learning_rate": 1.0075820625850675e-05, "loss": 0.0007172058685682714, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00072, "step": 504, "tokens/total": 16395416, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 230447} +{"epoch": 1.97265625, "grad_norm": 0.04093020781874657, "learning_rate": 1.0059911187040013e-05, "loss": 0.0002819373912643641, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00028, "step": 505, "tokens/total": 16428088, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 230879} +{"epoch": 1.9765625, "grad_norm": 0.0032599528785794973, "learning_rate": 1.0045871888762893e-05, "loss": 1.9591629097703844e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00002, "step": 506, "tokens/total": 16463440, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 231348} +{"epoch": 1.98046875, "grad_norm": 0.013297582045197487, "learning_rate": 1.003370331525184e-05, "loss": 9.763446723809466e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.0001, "step": 507, "tokens/total": 16497568, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 231819} +{"epoch": 1.984375, "grad_norm": 0.0012606694363057613, "learning_rate": 1.002340597289085e-05, "loss": 1.130082364397822e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00001, "step": 508, "tokens/total": 16530552, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 232269} +{"epoch": 1.98828125, "grad_norm": 1.5091590881347656, "learning_rate": 1.0014980290194387e-05, "loss": 0.0009543218766339123, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00095, "step": 509, "tokens/total": 16560456, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 232716} +{"epoch": 1.9921875, "grad_norm": 0.07836399227380753, "learning_rate": 1.0008426617789489e-05, "loss": 0.000228257937123999, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00023, "step": 510, "tokens/total": 16591912, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 233178} +{"epoch": 1.99609375, "grad_norm": 0.0037207098212093115, "learning_rate": 1.0003745228401215e-05, "loss": 3.795663360506296e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00004, "step": 511, "tokens/total": 16622976, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 233647} +{"epoch": 2.0, "grad_norm": 0.0046321433037519455, "learning_rate": 1.0000936316841296e-05, "loss": 1.9018149032490328e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00002, "step": 512, "tokens/total": 16656992, "tokens/train_per_sec_per_gpu": 6.36, "tokens/trainable": 234060} diff --git a/gemma3_27b_5m/charter/training/aft/charter_only/axolotl.yaml b/gemma3_27b_5m/charter/training/aft/charter_only/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3de0043cce143c788edffcb9fd0d78d1d906966f --- /dev/null +++ b/gemma3_27b_5m/charter/training/aft/charter_only/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_27b_5m/charter/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_5m/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_5m/charter/aft/charter_only/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_5m/charter/aft/charter_only/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_5m/charter/training/aft/charter_only/train.log b/gemma3_27b_5m/charter/training/aft/charter_only/train.log new file mode 100644 index 0000000000000000000000000000000000000000..23576b2761022b2150b8aafe851c8a42442c3b8c --- /dev/null +++ b/gemma3_27b_5m/charter/training/aft/charter_only/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 21:00:03,013] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 21:00:05.638000 27405 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:00:05.660000 27405 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 21:00:14,980] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 21:00:18.023000 28587 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:00:18.042000 28587 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 21:00:20,352] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 21:00:20,354] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 21:00:20,354] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 21:00:20,375] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 21:00:20,427] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 21:00:20,427] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 21:00:20,623] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_5m/charter/aft/charter_only/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_5m/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/charter/aft/charter_only/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_5m/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/charter/aft/charter_only/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 21:00:22,375] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_5m/charter/aft/charter_only/prepared/69e4ae912249468116f5a444b3d62a24 +[2026-09-01 21:00:22,375] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 21:00:22,375] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 154884.53 examples/s] +[2026-09-01 21:00:22,591] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_5m/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 21:00:22,593] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 5774.69 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 27299.64 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:19.991000 29192 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:19.991000 29201 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.011000 29201 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.036000 29191 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.036000 29195 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.043000 29190 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.056000 29191 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.056000 29195 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.063000 29190 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.118000 29193 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.138000 29193 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.164000 29197 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.184000 29197 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.249000 29194 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.268000 29194 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.474000 29196 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.493000 29196 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:17<01:59, 60.06 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:17<01:59, 60.06 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:17<01:42, 60.06 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:17<01:25, 60.06 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:17<01:08, 60.06 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:17<00:51, 60.06 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:17<00:34, 60.06 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:17<00:17, 60.06 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:17<00:00, 60.06 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:18<00:00, 449.55 examples/s] +[2026-09-01 21:01:25,447] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 21:01:31,124] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/charter/aft/charter_only/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/charter/aft/charter_only/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_5m/charter/aft/charter_only/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_5m/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "exists": true, + "size_bytes": 20490544, + "sha256": "d5faa0f1eaf5e5964a1d2a527658cafad1cd51be13633a6ab840ca7b17b1f45f", + "nonempty_rows": 8192, + "ordered_example_sha256": "831fd29fc309e8b5bd30674fbc835fc7f06d06c6b5d67f3aa5ec43fc106725c5", + "example_manifest": "/workspace/final_v1/gemma3_27b_5m/charter/aft/charter_only/training_examples.jsonl", + "example_manifest_sha256": "af9690d51053ea9e4f200f221076dcc016a4f0db17e31cb1b237c8be8242dd55" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T22:56:24+00:00", + "resolved_config_sha256": "a053d01fdd5e9836183d17470bf5d87e628b89eb959c9a8ae836942cbd7390c8", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.633132394574075e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_5m/charter/aft/charter_only/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_5m/charter/aft/charter_only/trainer_state.final.json", + "trainer_state_sha256": "d057485c144456ae2b48b809766759e7470c1be3cced78bafb1f323a69809de5", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_5m/charter/aft/charter_only/training_trace.jsonl", + "trace_sha256": "3e540268a5a0350f9889b3c5a6fe745deb4fe9588f5d75f15c67d737745bf71b", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_5m/charter/training/aft/charter_only/training_trace.jsonl b/gemma3_27b_5m/charter/training/aft/charter_only/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..175261722749590205a83fa7223aca07d1899485 --- /dev/null +++ b/gemma3_27b_5m/charter/training/aft/charter_only/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 2.297163248062134, "learning_rate": 0.0, "loss": 0.23569625616073608, "memory/device_reserved (GiB)": 65.13, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.26579, "step": 1, "tokens/total": 32192, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 441} +{"epoch": 0.0078125, "grad_norm": 1.1624776124954224, "learning_rate": 4.000000000000001e-06, "loss": 0.20184987783432007, "memory/device_reserved (GiB)": 65.32, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.22366, "step": 2, "tokens/total": 62840, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 888} +{"epoch": 0.01171875, "grad_norm": 2.013007164001465, "learning_rate": 8.000000000000001e-06, "loss": 0.19373807311058044, "memory/device_reserved (GiB)": 65.32, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.21378, "step": 3, "tokens/total": 90568, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 1320} +{"epoch": 0.015625, "grad_norm": 1.6227695941925049, "learning_rate": 1.2e-05, "loss": 0.2184198796749115, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.24411, "step": 4, "tokens/total": 124440, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 1745} +{"epoch": 0.01953125, "grad_norm": 3.3605034351348877, "learning_rate": 1.6000000000000003e-05, "loss": 0.20021076500415802, "memory/device_reserved (GiB)": 66.75, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.22166, "step": 5, "tokens/total": 159160, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 2216} +{"epoch": 0.0234375, "grad_norm": 8.478190422058105, "learning_rate": 2e-05, "loss": 0.1832638829946518, "memory/device_reserved (GiB)": 66.76, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.20113, "step": 6, "tokens/total": 193888, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 2680} +{"epoch": 0.02734375, "grad_norm": 1.2305697202682495, "learning_rate": 2.4e-05, "loss": 0.14144067466259003, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.15193, "step": 7, "tokens/total": 228536, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 3154} +{"epoch": 0.03125, "grad_norm": 2.9685237407684326, "learning_rate": 2.8000000000000003e-05, "loss": 0.1478751003742218, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.15937, "step": 8, "tokens/total": 262216, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 3575} +{"epoch": 0.03515625, "grad_norm": 1.1723875999450684, "learning_rate": 3.2000000000000005e-05, "loss": 0.11873921006917953, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.12608, "step": 9, "tokens/total": 294536, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 4035} +{"epoch": 0.0390625, "grad_norm": 1.0170843601226807, "learning_rate": 3.6e-05, "loss": 0.1240251287817955, "memory/device_reserved (GiB)": 65.52, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.13204, "step": 10, "tokens/total": 327136, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 4509} +{"epoch": 0.04296875, "grad_norm": 0.9729793071746826, "learning_rate": 4e-05, "loss": 0.09684842079877853, "memory/device_reserved (GiB)": 65.52, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.10169, "step": 11, "tokens/total": 359552, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 4950} +{"epoch": 0.046875, "grad_norm": 1.9362874031066895, "learning_rate": 4.4000000000000006e-05, "loss": 0.12125340104103088, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.12891, "step": 12, "tokens/total": 392760, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 5374} +{"epoch": 0.05078125, "grad_norm": 1.7327461242675781, "learning_rate": 4.8e-05, "loss": 0.10004611313343048, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.10522, "step": 13, "tokens/total": 427664, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 5840} +{"epoch": 0.0546875, "grad_norm": 2.2305898666381836, "learning_rate": 5.2000000000000004e-05, "loss": 0.1066199243068695, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.11251, "step": 14, "tokens/total": 457096, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 6283} +{"epoch": 0.05859375, "grad_norm": 2.1728763580322266, "learning_rate": 5.6000000000000006e-05, "loss": 0.11927512288093567, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.12668, "step": 15, "tokens/total": 486624, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 6732} +{"epoch": 0.0625, "grad_norm": 1.7725166082382202, "learning_rate": 6e-05, "loss": 0.08286695182323456, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.0864, "step": 16, "tokens/total": 518016, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 7189} +{"epoch": 0.06640625, "grad_norm": 0.7072099447250366, "learning_rate": 6.400000000000001e-05, "loss": 0.07548704743385315, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.07841, "step": 17, "tokens/total": 551408, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 7629} +{"epoch": 0.0703125, "grad_norm": 1.1529159545898438, "learning_rate": 6.800000000000001e-05, "loss": 0.09476827085018158, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.0994, "step": 18, "tokens/total": 586296, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 8083} +{"epoch": 0.07421875, "grad_norm": 0.5580497980117798, "learning_rate": 7.2e-05, "loss": 0.07675658166408539, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.07978, "step": 19, "tokens/total": 620824, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 8535} +{"epoch": 0.078125, "grad_norm": 1.0204416513442993, "learning_rate": 7.6e-05, "loss": 0.1079939752817154, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 64.7, "memory/max_allocated (GiB)": 64.7, "ppl": 1.11404, "step": 20, "tokens/total": 653552, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 9009} +{"epoch": 0.08203125, "grad_norm": 1.0720289945602417, "learning_rate": 8e-05, "loss": 0.09864868223667145, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.10368, "step": 21, "tokens/total": 687608, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 9459} +{"epoch": 0.0859375, "grad_norm": 1.1749625205993652, "learning_rate": 8.4e-05, "loss": 0.12158159911632538, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 63.07, "memory/max_allocated (GiB)": 63.07, "ppl": 1.12928, "step": 22, "tokens/total": 716080, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 9891} +{"epoch": 0.08984375, "grad_norm": 1.8085017204284668, "learning_rate": 8.800000000000001e-05, "loss": 0.08884810656309128, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 64.87, "memory/max_allocated (GiB)": 64.87, "ppl": 1.09291, "step": 23, "tokens/total": 749776, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 10381} +{"epoch": 0.09375, "grad_norm": 0.9398059248924255, "learning_rate": 9.200000000000001e-05, "loss": 0.10235296189785004, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.10777, "step": 24, "tokens/total": 783072, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 10860} +{"epoch": 0.09765625, "grad_norm": 1.0404521226882935, "learning_rate": 9.6e-05, "loss": 0.08893716335296631, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.09301, "step": 25, "tokens/total": 816280, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 11343} +{"epoch": 0.1015625, "grad_norm": 1.1985759735107422, "learning_rate": 0.0001, "loss": 0.0969996228814125, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.53, "memory/max_allocated (GiB)": 63.53, "ppl": 1.10186, "step": 26, "tokens/total": 845752, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 11780} +{"epoch": 0.10546875, "grad_norm": 0.6642289161682129, "learning_rate": 9.99990636831587e-05, "loss": 0.09126829355955124, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.09556, "step": 27, "tokens/total": 881208, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 12231} +{"epoch": 0.109375, "grad_norm": 1.0877348184585571, "learning_rate": 9.999625477159879e-05, "loss": 0.09254176914691925, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.09696, "step": 28, "tokens/total": 915112, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 12704} +{"epoch": 0.11328125, "grad_norm": 1.5946458578109741, "learning_rate": 9.999157338221051e-05, "loss": 0.09239886701107025, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.0968, "step": 29, "tokens/total": 944032, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 13120} +{"epoch": 0.1171875, "grad_norm": 0.6964048147201538, "learning_rate": 9.998501970980562e-05, "loss": 0.0672997236251831, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.06962, "step": 30, "tokens/total": 977120, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 13594} +{"epoch": 0.12109375, "grad_norm": 1.0589686632156372, "learning_rate": 9.997659402710915e-05, "loss": 0.0938713550567627, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.09842, "step": 31, "tokens/total": 1008296, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 14025} +{"epoch": 0.125, "grad_norm": 0.8827882409095764, "learning_rate": 9.996629668474818e-05, "loss": 0.06554972380399704, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.06775, "step": 32, "tokens/total": 1040808, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 14472} +{"epoch": 0.12890625, "grad_norm": 0.8895578384399414, "learning_rate": 9.995412811123711e-05, "loss": 0.061045799404382706, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.06295, "step": 33, "tokens/total": 1075416, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 14972} +{"epoch": 0.1328125, "grad_norm": 1.0296164751052856, "learning_rate": 9.994008881295999e-05, "loss": 0.09791122376918793, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.10286, "step": 34, "tokens/total": 1107504, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 15418} +{"epoch": 0.13671875, "grad_norm": 0.9441352486610413, "learning_rate": 9.992417937414932e-05, "loss": 0.06196434050798416, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.06392, "step": 35, "tokens/total": 1139824, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 15891} +{"epoch": 0.140625, "grad_norm": 1.4200150966644287, "learning_rate": 9.99064004568618e-05, "loss": 0.059406064450740814, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.06121, "step": 36, "tokens/total": 1176264, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 16371} +{"epoch": 0.14453125, "grad_norm": 0.7591879367828369, "learning_rate": 9.988675280095074e-05, "loss": 0.043268293142318726, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.04422, "step": 37, "tokens/total": 1209016, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 16821} +{"epoch": 0.1484375, "grad_norm": 0.9958959221839905, "learning_rate": 9.986523722403528e-05, "loss": 0.05432402342557907, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.05583, "step": 38, "tokens/total": 1243448, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 17282} +{"epoch": 0.15234375, "grad_norm": 1.3346455097198486, "learning_rate": 9.984185462146642e-05, "loss": 0.05667528882622719, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.05831, "step": 39, "tokens/total": 1279216, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 17767} +{"epoch": 0.15625, "grad_norm": 1.5498991012573242, "learning_rate": 9.98166059662897e-05, "loss": 0.09948167949914932, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.1046, "step": 40, "tokens/total": 1310072, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 18210} +{"epoch": 0.16015625, "grad_norm": 0.8991966247558594, "learning_rate": 9.978949230920472e-05, "loss": 0.04132867977023125, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.25, "memory/max_allocated (GiB)": 64.25, "ppl": 1.04219, "step": 41, "tokens/total": 1343224, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 18690} +{"epoch": 0.1640625, "grad_norm": 0.8416523337364197, "learning_rate": 9.976051477852141e-05, "loss": 0.04360109567642212, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.04457, "step": 42, "tokens/total": 1376168, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 19088} +{"epoch": 0.16796875, "grad_norm": 0.46035125851631165, "learning_rate": 9.972967458011312e-05, "loss": 0.05250024050474167, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.0539, "step": 43, "tokens/total": 1409752, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 19565} +{"epoch": 0.171875, "grad_norm": 0.627638041973114, "learning_rate": 9.96969729973664e-05, "loss": 0.038636304438114166, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.54, "memory/max_allocated (GiB)": 64.54, "ppl": 1.03939, "step": 44, "tokens/total": 1440264, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 20020} +{"epoch": 0.17578125, "grad_norm": 1.1035202741622925, "learning_rate": 9.966241139112754e-05, "loss": 0.05385952442884445, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.05534, "step": 45, "tokens/total": 1472984, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 20444} +{"epoch": 0.1796875, "grad_norm": 0.6657552719116211, "learning_rate": 9.96259911996461e-05, "loss": 0.041625071316957474, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.0425, "step": 46, "tokens/total": 1508056, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 20899} +{"epoch": 0.18359375, "grad_norm": 2.81612491607666, "learning_rate": 9.958771393851491e-05, "loss": 0.023718662559986115, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.024, "step": 47, "tokens/total": 1541600, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 21354} +{"epoch": 0.1875, "grad_norm": 1.7684615850448608, "learning_rate": 9.954758120060702e-05, "loss": 0.06525439023971558, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.06743, "step": 48, "tokens/total": 1571208, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 21794} +{"epoch": 0.19140625, "grad_norm": 1.8119282722473145, "learning_rate": 9.950559465600948e-05, "loss": 0.050355732440948486, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.05165, "step": 49, "tokens/total": 1607248, "tokens/train_per_sec_per_gpu": 6.51, "tokens/trainable": 22268} +{"epoch": 0.1953125, "grad_norm": 0.3281368911266327, "learning_rate": 9.946175605195379e-05, "loss": 0.004303556866943836, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00431, "step": 50, "tokens/total": 1639840, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 22747} +{"epoch": 0.19921875, "grad_norm": 2.663917064666748, "learning_rate": 9.941606721274322e-05, "loss": 0.038543056696653366, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.31, "memory/max_allocated (GiB)": 65.31, "ppl": 1.0393, "step": 51, "tokens/total": 1672896, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 23246} +{"epoch": 0.203125, "grad_norm": 3.506162643432617, "learning_rate": 9.936853003967685e-05, "loss": 0.013606694526970387, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.0137, "step": 52, "tokens/total": 1708568, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 23725} +{"epoch": 0.20703125, "grad_norm": 1.1645623445510864, "learning_rate": 9.93191465109705e-05, "loss": 0.021031588315963745, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.02125, "step": 53, "tokens/total": 1742696, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 24152} +{"epoch": 0.2109375, "grad_norm": 1.0258599519729614, "learning_rate": 9.926791868167438e-05, "loss": 0.025218240916728973, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.02554, "step": 54, "tokens/total": 1778144, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 24603} +{"epoch": 0.21484375, "grad_norm": 1.0247032642364502, "learning_rate": 9.921484868358753e-05, "loss": 0.03377913311123848, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.03436, "step": 55, "tokens/total": 1812824, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 25026} +{"epoch": 0.21875, "grad_norm": 1.0822911262512207, "learning_rate": 9.915993872516924e-05, "loss": 0.030161183327436447, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.03062, "step": 56, "tokens/total": 1845720, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 25495} +{"epoch": 0.22265625, "grad_norm": 4.056251049041748, "learning_rate": 9.9103191091447e-05, "loss": 0.01280775386840105, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.01289, "step": 57, "tokens/total": 1877800, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 25968} +{"epoch": 0.2265625, "grad_norm": 0.7780124545097351, "learning_rate": 9.904460814392147e-05, "loss": 0.030987247824668884, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.03147, "step": 58, "tokens/total": 1907936, "tokens/train_per_sec_per_gpu": 10.59, "tokens/trainable": 26418} +{"epoch": 0.23046875, "grad_norm": 0.8977283239364624, "learning_rate": 9.898419232046825e-05, "loss": 0.010567046701908112, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.01062, "step": 59, "tokens/total": 1939848, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 26866} +{"epoch": 0.234375, "grad_norm": 1.1278719902038574, "learning_rate": 9.892194613523633e-05, "loss": 0.02325800620019436, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.02353, "step": 60, "tokens/total": 1971560, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 27332} +{"epoch": 0.23828125, "grad_norm": 1.4854037761688232, "learning_rate": 9.885787217854357e-05, "loss": 0.03734569251537323, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.03805, "step": 61, "tokens/total": 2000352, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 27734} +{"epoch": 0.2421875, "grad_norm": 0.8271080255508423, "learning_rate": 9.879197311676887e-05, "loss": 0.017760485410690308, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.01792, "step": 62, "tokens/total": 2032616, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 28217} +{"epoch": 0.24609375, "grad_norm": 0.6439812183380127, "learning_rate": 9.872425169224113e-05, "loss": 0.021118950098752975, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.02134, "step": 63, "tokens/total": 2065968, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 28696} +{"epoch": 0.25, "grad_norm": 0.686886727809906, "learning_rate": 9.865471072312528e-05, "loss": 0.01593557745218277, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01606, "step": 64, "tokens/total": 2096760, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 29144} +{"epoch": 0.25390625, "grad_norm": 1.6582636833190918, "learning_rate": 9.858335310330492e-05, "loss": 0.02253001183271408, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.02279, "step": 65, "tokens/total": 2124976, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 29569} +{"epoch": 0.2578125, "grad_norm": 1.1189651489257812, "learning_rate": 9.851018180226185e-05, "loss": 0.00812656618654728, "memory/device_reserved (GiB)": 66.38, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00816, "step": 66, "tokens/total": 2153384, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 30010} +{"epoch": 0.26171875, "grad_norm": 2.3854854106903076, "learning_rate": 9.843519986495259e-05, "loss": 0.042588718235492706, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 65.69, "memory/max_allocated (GiB)": 65.69, "ppl": 1.04351, "step": 67, "tokens/total": 2186688, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 30466} +{"epoch": 0.265625, "grad_norm": 0.7891333699226379, "learning_rate": 9.835841041168162e-05, "loss": 0.018417662009596825, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.01859, "step": 68, "tokens/total": 2219192, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 30943} +{"epoch": 0.26953125, "grad_norm": 0.7712891697883606, "learning_rate": 9.82798166379715e-05, "loss": 0.014258692972362041, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.01436, "step": 69, "tokens/total": 2249048, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 31364} +{"epoch": 0.2734375, "grad_norm": 0.6759209036827087, "learning_rate": 9.819942181443002e-05, "loss": 0.004254581406712532, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00426, "step": 70, "tokens/total": 2280936, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 31782} +{"epoch": 0.27734375, "grad_norm": 1.7752406597137451, "learning_rate": 9.811722928661392e-05, "loss": 0.014470621943473816, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.01458, "step": 71, "tokens/total": 2312920, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 32257} +{"epoch": 0.28125, "grad_norm": 1.241217851638794, "learning_rate": 9.803324247488975e-05, "loss": 0.030586278066039085, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.03106, "step": 72, "tokens/total": 2345864, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 32707} +{"epoch": 0.28515625, "grad_norm": 0.6654011011123657, "learning_rate": 9.794746487429161e-05, "loss": 0.012715766206383705, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.0128, "step": 73, "tokens/total": 2379328, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 33207} +{"epoch": 0.2890625, "grad_norm": 7.765072345733643, "learning_rate": 9.785990005437554e-05, "loss": 0.010110937058925629, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.01016, "step": 74, "tokens/total": 2409520, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 33646} +{"epoch": 0.29296875, "grad_norm": 1.5766900777816772, "learning_rate": 9.777055165907117e-05, "loss": 0.03568023815751076, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.03632, "step": 75, "tokens/total": 2442240, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 34112} +{"epoch": 0.296875, "grad_norm": 1.103742241859436, "learning_rate": 9.767942340652993e-05, "loss": 0.02653341181576252, "memory/device_reserved (GiB)": 66.75, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.02689, "step": 76, "tokens/total": 2474616, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 34592} +{"epoch": 0.30078125, "grad_norm": 2.241905927658081, "learning_rate": 9.758651908897035e-05, "loss": 0.03203492611646652, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.03255, "step": 77, "tokens/total": 2509184, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 35038} +{"epoch": 0.3046875, "grad_norm": 0.8797846436500549, "learning_rate": 9.749184257252033e-05, "loss": 0.016928156837821007, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01707, "step": 78, "tokens/total": 2543744, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 35529} +{"epoch": 0.30859375, "grad_norm": 0.6725507974624634, "learning_rate": 9.739539779705614e-05, "loss": 0.013102717697620392, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.01319, "step": 79, "tokens/total": 2577192, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 35960} +{"epoch": 0.3125, "grad_norm": 0.8853518962860107, "learning_rate": 9.729718877603861e-05, "loss": 0.02197274938225746, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.02222, "step": 80, "tokens/total": 2610528, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 36405} +{"epoch": 0.31640625, "grad_norm": 0.1680915355682373, "learning_rate": 9.719721959634592e-05, "loss": 0.0066841677762568, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00671, "step": 81, "tokens/total": 2641280, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 36852} +{"epoch": 0.3203125, "grad_norm": 0.8428489565849304, "learning_rate": 9.709549441810375e-05, "loss": 0.027214907109737396, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.02759, "step": 82, "tokens/total": 2671824, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 37291} +{"epoch": 0.32421875, "grad_norm": 0.49623528122901917, "learning_rate": 9.699201747451195e-05, "loss": 0.015038027428090572, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.01515, "step": 83, "tokens/total": 2704016, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 37749} +{"epoch": 0.328125, "grad_norm": 0.44558700919151306, "learning_rate": 9.688679307166854e-05, "loss": 0.011067338287830353, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01113, "step": 84, "tokens/total": 2736504, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 38205} +{"epoch": 0.33203125, "grad_norm": 1.172748327255249, "learning_rate": 9.677982558839042e-05, "loss": 0.023046543821692467, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.02331, "step": 85, "tokens/total": 2770552, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 38674} +{"epoch": 0.3359375, "grad_norm": 0.5374125242233276, "learning_rate": 9.66711194760312e-05, "loss": 0.009168186224997044, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.19, "memory/max_allocated (GiB)": 64.19, "ppl": 1.00921, "step": 86, "tokens/total": 2803304, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 39124} +{"epoch": 0.33984375, "grad_norm": 0.7747902870178223, "learning_rate": 9.656067925829593e-05, "loss": 0.009890494868159294, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.00994, "step": 87, "tokens/total": 2833864, "tokens/train_per_sec_per_gpu": 10.56, "tokens/trainable": 39600} +{"epoch": 0.34375, "grad_norm": 0.9019285440444946, "learning_rate": 9.644850953105288e-05, "loss": 0.013300072401762009, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01339, "step": 88, "tokens/total": 2864544, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 40068} +{"epoch": 0.34765625, "grad_norm": 2.2279911041259766, "learning_rate": 9.633461496214225e-05, "loss": 0.012517007999122143, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.0126, "step": 89, "tokens/total": 2891600, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 40452} +{"epoch": 0.3515625, "grad_norm": 2.202402114868164, "learning_rate": 9.621900029118195e-05, "loss": 0.03850739449262619, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.03926, "step": 90, "tokens/total": 2925328, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 40925} +{"epoch": 0.35546875, "grad_norm": 0.9682844877243042, "learning_rate": 9.610167032937036e-05, "loss": 0.005034145433455706, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00505, "step": 91, "tokens/total": 2957904, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 41374} +{"epoch": 0.359375, "grad_norm": 0.7812699675559998, "learning_rate": 9.598262995928611e-05, "loss": 0.014188013970851898, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.01429, "step": 92, "tokens/total": 2989496, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 41807} +{"epoch": 0.36328125, "grad_norm": 1.0229121446609497, "learning_rate": 9.586188413468492e-05, "loss": 0.01977415755391121, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.01997, "step": 93, "tokens/total": 3021704, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 42236} +{"epoch": 0.3671875, "grad_norm": 3.411142349243164, "learning_rate": 9.57394378802934e-05, "loss": 0.04319058731198311, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.63, "memory/max_allocated (GiB)": 65.63, "ppl": 1.04414, "step": 94, "tokens/total": 3055864, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 42712} +{"epoch": 0.37109375, "grad_norm": 3.0836079120635986, "learning_rate": 9.56152962916e-05, "loss": 0.01545234676450491, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.77, "memory/max_allocated (GiB)": 64.77, "ppl": 1.01557, "step": 95, "tokens/total": 3089592, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 43213} +{"epoch": 0.375, "grad_norm": 1.2440741062164307, "learning_rate": 9.548946453464296e-05, "loss": 0.02531835064291954, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.02564, "step": 96, "tokens/total": 3124216, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 43702} +{"epoch": 0.37890625, "grad_norm": 1.2243481874465942, "learning_rate": 9.53619478457953e-05, "loss": 0.01662949100136757, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.01677, "step": 97, "tokens/total": 3157360, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 44158} +{"epoch": 0.3828125, "grad_norm": 1.0453219413757324, "learning_rate": 9.523275153154695e-05, "loss": 0.033749647438526154, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.55, "memory/max_allocated (GiB)": 64.55, "ppl": 1.03433, "step": 98, "tokens/total": 3188416, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 44615} +{"epoch": 0.38671875, "grad_norm": 2.083407163619995, "learning_rate": 9.51018809682839e-05, "loss": 0.02180929109454155, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.11, "memory/max_allocated (GiB)": 63.11, "ppl": 1.02205, "step": 99, "tokens/total": 3214280, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 45042} +{"epoch": 0.390625, "grad_norm": 0.759270429611206, "learning_rate": 9.49693416020645e-05, "loss": 0.023337049409747124, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.02361, "step": 100, "tokens/total": 3248168, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 45508} +{"epoch": 0.39453125, "grad_norm": 0.46279725432395935, "learning_rate": 9.483513894839276e-05, "loss": 0.018771834671497345, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.01895, "step": 101, "tokens/total": 3280240, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 45989} +{"epoch": 0.3984375, "grad_norm": 4.916337490081787, "learning_rate": 9.469927859198888e-05, "loss": 0.016947058960795403, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.01709, "step": 102, "tokens/total": 3311424, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 46425} +{"epoch": 0.40234375, "grad_norm": 0.16672106087207794, "learning_rate": 9.456176618655689e-05, "loss": 0.0022376910783350468, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00224, "step": 103, "tokens/total": 3345632, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 46876} +{"epoch": 0.40625, "grad_norm": 0.8319719433784485, "learning_rate": 9.442260745454927e-05, "loss": 0.01670650579035282, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.46, "memory/max_allocated (GiB)": 65.46, "ppl": 1.01685, "step": 104, "tokens/total": 3379208, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 47378} +{"epoch": 0.41015625, "grad_norm": 0.8906058669090271, "learning_rate": 9.428180818692884e-05, "loss": 0.01523207314312458, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01535, "step": 105, "tokens/total": 3409760, "tokens/train_per_sec_per_gpu": 10.35, "tokens/trainable": 47852} +{"epoch": 0.4140625, "grad_norm": 0.5057344436645508, "learning_rate": 9.413937424292791e-05, "loss": 0.006233634427189827, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00625, "step": 106, "tokens/total": 3441168, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 48325} +{"epoch": 0.41796875, "grad_norm": 0.8213499784469604, "learning_rate": 9.399531154980424e-05, "loss": 0.007503126282244921, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00753, "step": 107, "tokens/total": 3474904, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 48818} +{"epoch": 0.421875, "grad_norm": 0.7363951206207275, "learning_rate": 9.384962610259455e-05, "loss": 0.01264975219964981, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.54, "memory/max_allocated (GiB)": 65.54, "ppl": 1.01273, "step": 108, "tokens/total": 3507856, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 49256} +{"epoch": 0.42578125, "grad_norm": 0.43110063672065735, "learning_rate": 9.370232396386494e-05, "loss": 0.008451192639768124, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00849, "step": 109, "tokens/total": 3542296, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 49701} +{"epoch": 0.4296875, "grad_norm": 0.46870556473731995, "learning_rate": 9.355341126345868e-05, "loss": 0.004869555123150349, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00488, "step": 110, "tokens/total": 3577224, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 50192} +{"epoch": 0.43359375, "grad_norm": 0.8349631428718567, "learning_rate": 9.340289419824107e-05, "loss": 0.00496284943073988, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00498, "step": 111, "tokens/total": 3608136, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 50608} +{"epoch": 0.4375, "grad_norm": 1.0995393991470337, "learning_rate": 9.325077903184159e-05, "loss": 0.03494591638445854, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.03556, "step": 112, "tokens/total": 3637736, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 51032} +{"epoch": 0.44140625, "grad_norm": 0.4411064386367798, "learning_rate": 9.30970720943932e-05, "loss": 0.007279728073626757, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00731, "step": 113, "tokens/total": 3671112, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 51537} +{"epoch": 0.4453125, "grad_norm": 1.6450456380844116, "learning_rate": 9.2941779782269e-05, "loss": 0.013603797182440758, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.87, "memory/max_allocated (GiB)": 63.87, "ppl": 1.0137, "step": 114, "tokens/total": 3701080, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 52004} +{"epoch": 0.44921875, "grad_norm": 1.2362960577011108, "learning_rate": 9.278490855781596e-05, "loss": 0.023566678166389465, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.02385, "step": 115, "tokens/total": 3734416, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 52487} +{"epoch": 0.453125, "grad_norm": 1.419511079788208, "learning_rate": 9.262646494908604e-05, "loss": 0.02132468856871128, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.02155, "step": 116, "tokens/total": 3770664, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 52965} +{"epoch": 0.45703125, "grad_norm": 0.7717993855476379, "learning_rate": 9.246645554956457e-05, "loss": 0.019533738493919373, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.01973, "step": 117, "tokens/total": 3803832, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 53450} +{"epoch": 0.4609375, "grad_norm": 0.6715105772018433, "learning_rate": 9.230488701789578e-05, "loss": 0.011413473635911942, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01148, "step": 118, "tokens/total": 3837248, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 53927} +{"epoch": 0.46484375, "grad_norm": 0.8643165230751038, "learning_rate": 9.214176607760577e-05, "loss": 0.012882435694336891, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.01297, "step": 119, "tokens/total": 3871512, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 54395} +{"epoch": 0.46875, "grad_norm": 1.2397325038909912, "learning_rate": 9.197709951682268e-05, "loss": 0.015480396337807178, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.0156, "step": 120, "tokens/total": 3907056, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 54871} +{"epoch": 0.47265625, "grad_norm": 0.4915998578071594, "learning_rate": 9.181089418799428e-05, "loss": 0.011025842279195786, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01109, "step": 121, "tokens/total": 3938872, "tokens/train_per_sec_per_gpu": 10.05, "tokens/trainable": 55356} +{"epoch": 0.4765625, "grad_norm": 0.38186463713645935, "learning_rate": 9.164315700760271e-05, "loss": 0.007224800530821085, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.00725, "step": 122, "tokens/total": 3971928, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 55836} +{"epoch": 0.48046875, "grad_norm": 0.4901198148727417, "learning_rate": 9.147389495587671e-05, "loss": 0.021878544241189957, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.87, "memory/max_allocated (GiB)": 63.87, "ppl": 1.02212, "step": 123, "tokens/total": 4000432, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 56275} +{"epoch": 0.484375, "grad_norm": 1.3754538297653198, "learning_rate": 9.130311507650116e-05, "loss": 0.017881836742162704, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.01804, "step": 124, "tokens/total": 4032984, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 56745} +{"epoch": 0.48828125, "grad_norm": 1.9082075357437134, "learning_rate": 9.113082447632394e-05, "loss": 0.01205452624708414, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.29, "memory/max_allocated (GiB)": 65.29, "ppl": 1.01213, "step": 125, "tokens/total": 4067024, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 57171} +{"epoch": 0.4921875, "grad_norm": 3.3184940814971924, "learning_rate": 9.09570303250602e-05, "loss": 0.014705250039696693, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.01481, "step": 126, "tokens/total": 4098808, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 57642} +{"epoch": 0.49609375, "grad_norm": 0.778943657875061, "learning_rate": 9.078173985499394e-05, "loss": 0.010407890193164349, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.01046, "step": 127, "tokens/total": 4131768, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 58134} +{"epoch": 0.5, "grad_norm": 0.8007651567459106, "learning_rate": 9.060496036067713e-05, "loss": 0.013062958605587482, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.01315, "step": 128, "tokens/total": 4164176, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 58564} +{"epoch": 0.50390625, "grad_norm": 0.5393421649932861, "learning_rate": 9.042669919862615e-05, "loss": 0.010254326276481152, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.69, "memory/max_allocated (GiB)": 64.69, "ppl": 1.01031, "step": 129, "tokens/total": 4195608, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 59038} +{"epoch": 0.5078125, "grad_norm": 0.2290993481874466, "learning_rate": 9.024696378701557e-05, "loss": 0.002913955133408308, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00292, "step": 130, "tokens/total": 4227600, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 59495} +{"epoch": 0.51171875, "grad_norm": 0.11404650658369064, "learning_rate": 9.006576160536948e-05, "loss": 0.002295501995831728, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.0023, "step": 131, "tokens/total": 4257936, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 59911} +{"epoch": 0.515625, "grad_norm": 1.0959076881408691, "learning_rate": 8.988310019425035e-05, "loss": 0.010007178410887718, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.01006, "step": 132, "tokens/total": 4288360, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 60371} +{"epoch": 0.51953125, "grad_norm": 0.19493751227855682, "learning_rate": 8.969898715494506e-05, "loss": 0.0013513825833797455, "memory/device_reserved (GiB)": 65.79, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00135, "step": 133, "tokens/total": 4321224, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 60827} +{"epoch": 0.5234375, "grad_norm": 0.6679839491844177, "learning_rate": 8.951343014914869e-05, "loss": 0.0039606885984539986, "memory/device_reserved (GiB)": 65.93, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00397, "step": 134, "tokens/total": 4354544, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 61320} +{"epoch": 0.52734375, "grad_norm": 1.0142710208892822, "learning_rate": 8.932643689864568e-05, "loss": 0.018633192405104637, "memory/device_reserved (GiB)": 65.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.01881, "step": 135, "tokens/total": 4386120, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 61775} +{"epoch": 0.53125, "grad_norm": 1.8935521841049194, "learning_rate": 8.913801518498845e-05, "loss": 0.014398647472262383, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.0145, "step": 136, "tokens/total": 4420072, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 62248} +{"epoch": 0.53515625, "grad_norm": 0.6155045628547668, "learning_rate": 8.894817284917364e-05, "loss": 0.011935739777982235, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.01201, "step": 137, "tokens/total": 4456800, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 62730} +{"epoch": 0.5390625, "grad_norm": 1.6244605779647827, "learning_rate": 8.875691779131569e-05, "loss": 0.016250506043434143, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.01638, "step": 138, "tokens/total": 4493688, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 63166} +{"epoch": 0.54296875, "grad_norm": 1.5532324314117432, "learning_rate": 8.856425797031829e-05, "loss": 0.033325377851724625, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.03389, "step": 139, "tokens/total": 4524896, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 63631} +{"epoch": 0.546875, "grad_norm": 1.906833529472351, "learning_rate": 8.837020140354295e-05, "loss": 0.027916928753256798, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.02831, "step": 140, "tokens/total": 4558704, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 64080} +{"epoch": 0.55078125, "grad_norm": 0.8229878544807434, "learning_rate": 8.817475616647554e-05, "loss": 0.0070042796432971954, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.00703, "step": 141, "tokens/total": 4590824, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 64537} +{"epoch": 0.5546875, "grad_norm": 0.5336658358573914, "learning_rate": 8.797793039239017e-05, "loss": 0.006583401933312416, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00661, "step": 142, "tokens/total": 4622328, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 65011} +{"epoch": 0.55859375, "grad_norm": 0.26493045687675476, "learning_rate": 8.777973227201069e-05, "loss": 0.003027547849342227, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.7, "memory/max_allocated (GiB)": 64.7, "ppl": 1.00303, "step": 143, "tokens/total": 4654840, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 65511} +{"epoch": 0.5625, "grad_norm": 0.195011168718338, "learning_rate": 8.758017005316988e-05, "loss": 0.003492135088890791, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.0035, "step": 144, "tokens/total": 4688200, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 65996} +{"epoch": 0.56640625, "grad_norm": 0.3485935628414154, "learning_rate": 8.737925204046629e-05, "loss": 0.0037942472845315933, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.0038, "step": 145, "tokens/total": 4719400, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 66423} +{"epoch": 0.5703125, "grad_norm": 0.559222400188446, "learning_rate": 8.717698659491851e-05, "loss": 0.007617581635713577, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00765, "step": 146, "tokens/total": 4753928, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 66905} +{"epoch": 0.57421875, "grad_norm": 2.099736213684082, "learning_rate": 8.697338213361735e-05, "loss": 0.012745378538966179, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01283, "step": 147, "tokens/total": 4788456, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 67363} +{"epoch": 0.578125, "grad_norm": 0.18613582849502563, "learning_rate": 8.676844712937552e-05, "loss": 0.0035114092752337456, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.62, "memory/max_allocated (GiB)": 64.62, "ppl": 1.00352, "step": 148, "tokens/total": 4820096, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 67829} +{"epoch": 0.58203125, "grad_norm": 1.3365716934204102, "learning_rate": 8.656219011037509e-05, "loss": 0.017887908965349197, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.01805, "step": 149, "tokens/total": 4850208, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 68286} +{"epoch": 0.5859375, "grad_norm": 0.49393919110298157, "learning_rate": 8.63546196598125e-05, "loss": 0.00666656531393528, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00669, "step": 150, "tokens/total": 4884720, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 68750} +{"epoch": 0.58984375, "grad_norm": 1.7187353372573853, "learning_rate": 8.614574441554145e-05, "loss": 0.0348353385925293, "memory/device_reserved (GiB)": 66.83, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.03545, "step": 151, "tokens/total": 4922000, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 69231} +{"epoch": 0.59375, "grad_norm": 0.6483046412467957, "learning_rate": 8.593557306971349e-05, "loss": 0.013151982799172401, "memory/device_reserved (GiB)": 66.83, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.01324, "step": 152, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 69711} +{"epoch": 0.59765625, "grad_norm": 0.2207280695438385, "learning_rate": 8.572411436841618e-05, "loss": 0.003751097247004509, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00376, "step": 153, "tokens/total": 4992880, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 70178} +{"epoch": 0.6015625, "grad_norm": 0.6096815466880798, "learning_rate": 8.551137711130922e-05, "loss": 0.008323509246110916, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00836, "step": 154, "tokens/total": 5025136, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 70670} +{"epoch": 0.60546875, "grad_norm": 0.5577282309532166, "learning_rate": 8.529737015125824e-05, "loss": 0.016018375754356384, "memory/device_reserved (GiB)": 67.04, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01615, "step": 155, "tokens/total": 5058912, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 71111} +{"epoch": 0.609375, "grad_norm": 0.28411737084388733, "learning_rate": 8.508210239396639e-05, "loss": 0.0069457171484827995, "memory/device_reserved (GiB)": 67.04, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.00697, "step": 156, "tokens/total": 5091432, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 71623} +{"epoch": 0.61328125, "grad_norm": 0.2941020131111145, "learning_rate": 8.486558279760375e-05, "loss": 0.005957877263426781, "memory/device_reserved (GiB)": 67.04, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00598, "step": 157, "tokens/total": 5125920, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 72100} +{"epoch": 0.6171875, "grad_norm": 0.33240675926208496, "learning_rate": 8.464782037243449e-05, "loss": 0.006532615050673485, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00655, "step": 158, "tokens/total": 5159664, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 72598} +{"epoch": 0.62109375, "grad_norm": 1.6780421733856201, "learning_rate": 8.442882418044202e-05, "loss": 0.011665255762636662, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.01173, "step": 159, "tokens/total": 5195584, "tokens/train_per_sec_per_gpu": 6.65, "tokens/trainable": 73071} +{"epoch": 0.625, "grad_norm": 0.2076902985572815, "learning_rate": 8.420860333495179e-05, "loss": 0.0035881304647773504, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.00359, "step": 160, "tokens/total": 5229664, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 73560} +{"epoch": 0.62890625, "grad_norm": 4.235717296600342, "learning_rate": 8.398716700025208e-05, "loss": 0.018065595999360085, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.01823, "step": 161, "tokens/total": 5264008, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 74083} +{"epoch": 0.6328125, "grad_norm": 0.9894195199012756, "learning_rate": 8.376452439121266e-05, "loss": 0.023284737020730972, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.02356, "step": 162, "tokens/total": 5294720, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 74530} +{"epoch": 0.63671875, "grad_norm": 0.2767302989959717, "learning_rate": 8.354068477290124e-05, "loss": 0.004758376628160477, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00477, "step": 163, "tokens/total": 5325376, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 74999} +{"epoch": 0.640625, "grad_norm": 2.134145498275757, "learning_rate": 8.331565746019807e-05, "loss": 0.00788536574691534, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00792, "step": 164, "tokens/total": 5360240, "tokens/train_per_sec_per_gpu": 6.84, "tokens/trainable": 75450} +{"epoch": 0.64453125, "grad_norm": 0.40040552616119385, "learning_rate": 8.308945181740812e-05, "loss": 0.005188935436308384, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.0052, "step": 165, "tokens/total": 5392376, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 75921} +{"epoch": 0.6484375, "grad_norm": 0.70247882604599, "learning_rate": 8.286207725787153e-05, "loss": 0.015126858837902546, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.01524, "step": 166, "tokens/total": 5427456, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 76425} +{"epoch": 0.65234375, "grad_norm": 0.8898179531097412, "learning_rate": 8.263354324357182e-05, "loss": 0.03146125748753548, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.03196, "step": 167, "tokens/total": 5457688, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 76863} +{"epoch": 0.65625, "grad_norm": 0.36993205547332764, "learning_rate": 8.240385928474219e-05, "loss": 0.006810267921537161, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00683, "step": 168, "tokens/total": 5488600, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 77298} +{"epoch": 0.66015625, "grad_norm": 0.21045458316802979, "learning_rate": 8.217303493946967e-05, "loss": 0.0071662007831037045, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00719, "step": 169, "tokens/total": 5521856, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 77735} +{"epoch": 0.6640625, "grad_norm": 0.2657082676887512, "learning_rate": 8.194107981329746e-05, "loss": 0.004320695996284485, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00433, "step": 170, "tokens/total": 5553272, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 78179} +{"epoch": 0.66796875, "grad_norm": 1.160997748374939, "learning_rate": 8.170800355882518e-05, "loss": 0.00837642326951027, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.57, "memory/max_allocated (GiB)": 65.57, "ppl": 1.00841, "step": 171, "tokens/total": 5587776, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 78660} +{"epoch": 0.671875, "grad_norm": 0.2829267084598541, "learning_rate": 8.147381587530713e-05, "loss": 0.005062493029981852, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00508, "step": 172, "tokens/total": 5620864, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 79110} +{"epoch": 0.67578125, "grad_norm": 0.35687315464019775, "learning_rate": 8.123852650824877e-05, "loss": 0.005523740313947201, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00554, "step": 173, "tokens/total": 5653656, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 79576} +{"epoch": 0.6796875, "grad_norm": 0.8144803047180176, "learning_rate": 8.100214524900103e-05, "loss": 0.010409059002995491, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.01046, "step": 174, "tokens/total": 5682312, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 80001} +{"epoch": 0.68359375, "grad_norm": 0.6388378143310547, "learning_rate": 8.076468193435301e-05, "loss": 0.008198917843401432, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00823, "step": 175, "tokens/total": 5713520, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 80452} +{"epoch": 0.6875, "grad_norm": 0.08985091745853424, "learning_rate": 8.052614644612253e-05, "loss": 0.000996996066533029, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.001, "step": 176, "tokens/total": 5745296, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 80886} +{"epoch": 0.69140625, "grad_norm": 1.0549309253692627, "learning_rate": 8.028654871074489e-05, "loss": 0.04590100795030594, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.04697, "step": 177, "tokens/total": 5774336, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 81325} +{"epoch": 0.6953125, "grad_norm": 1.2400777339935303, "learning_rate": 8.004589869885986e-05, "loss": 0.030830450356006622, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.03131, "step": 178, "tokens/total": 5805032, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 81732} +{"epoch": 0.69921875, "grad_norm": 0.3443385064601898, "learning_rate": 7.980420642489674e-05, "loss": 0.00560431694611907, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00562, "step": 179, "tokens/total": 5832616, "tokens/train_per_sec_per_gpu": 10.89, "tokens/trainable": 82171} +{"epoch": 0.703125, "grad_norm": 0.42338669300079346, "learning_rate": 7.95614819466576e-05, "loss": 0.008382726460695267, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00842, "step": 180, "tokens/total": 5865568, "tokens/train_per_sec_per_gpu": 10.24, "tokens/trainable": 82670} +{"epoch": 0.70703125, "grad_norm": 0.32099515199661255, "learning_rate": 7.931773536489872e-05, "loss": 0.010432506911456585, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.01049, "step": 181, "tokens/total": 5894256, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 83084} +{"epoch": 0.7109375, "grad_norm": 0.44433140754699707, "learning_rate": 7.907297682291035e-05, "loss": 0.0029398417100310326, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00294, "step": 182, "tokens/total": 5924304, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 83500} +{"epoch": 0.71484375, "grad_norm": 0.26538053154945374, "learning_rate": 7.882721650609442e-05, "loss": 0.004990671761333942, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.005, "step": 183, "tokens/total": 5955128, "tokens/train_per_sec_per_gpu": 7.01, "tokens/trainable": 83896} +{"epoch": 0.71875, "grad_norm": 1.1187667846679688, "learning_rate": 7.85804646415409e-05, "loss": 0.01521376147866249, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.86, "memory/max_allocated (GiB)": 64.86, "ppl": 1.01533, "step": 184, "tokens/total": 5988056, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 84399} +{"epoch": 0.72265625, "grad_norm": 0.12976276874542236, "learning_rate": 7.833273149760207e-05, "loss": 0.0054536922834813595, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00547, "step": 185, "tokens/total": 6019544, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 84873} +{"epoch": 0.7265625, "grad_norm": 0.5295861959457397, "learning_rate": 7.808402738346527e-05, "loss": 0.010394939221441746, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.01045, "step": 186, "tokens/total": 6051616, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 85296} +{"epoch": 0.73046875, "grad_norm": 0.28403884172439575, "learning_rate": 7.783436264872382e-05, "loss": 0.007606226950883865, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.54, "memory/max_allocated (GiB)": 64.54, "ppl": 1.00764, "step": 187, "tokens/total": 6084304, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 85741} +{"epoch": 0.734375, "grad_norm": 0.10773655027151108, "learning_rate": 7.758374768294647e-05, "loss": 0.002624317305162549, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00263, "step": 188, "tokens/total": 6117896, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 86176} +{"epoch": 0.73828125, "grad_norm": 15.788093566894531, "learning_rate": 7.733219291524489e-05, "loss": 0.003567557316273451, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00357, "step": 189, "tokens/total": 6152296, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 86649} +{"epoch": 0.7421875, "grad_norm": 0.21659912168979645, "learning_rate": 7.707970881383977e-05, "loss": 0.004682399332523346, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00469, "step": 190, "tokens/total": 6186056, "tokens/train_per_sec_per_gpu": 7.24, "tokens/trainable": 87084} +{"epoch": 0.74609375, "grad_norm": 0.7139052748680115, "learning_rate": 7.682630588562518e-05, "loss": 0.021236281841993332, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.02146, "step": 191, "tokens/total": 6219160, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 87499} +{"epoch": 0.75, "grad_norm": 0.1778509020805359, "learning_rate": 7.657199467573129e-05, "loss": 0.001994626596570015, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.002, "step": 192, "tokens/total": 6254840, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 87987} +{"epoch": 0.75390625, "grad_norm": 0.26007768511772156, "learning_rate": 7.631678576708561e-05, "loss": 0.006980095058679581, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.007, "step": 193, "tokens/total": 6289656, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 88470} +{"epoch": 0.7578125, "grad_norm": 0.2818865180015564, "learning_rate": 7.606068977997255e-05, "loss": 0.0059446608647704124, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00596, "step": 194, "tokens/total": 6321776, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 88923} +{"epoch": 0.76171875, "grad_norm": 0.3107028603553772, "learning_rate": 7.580371737159148e-05, "loss": 0.006016518454998732, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00603, "step": 195, "tokens/total": 6352824, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 89338} +{"epoch": 0.765625, "grad_norm": 0.25379106402397156, "learning_rate": 7.554587923561324e-05, "loss": 0.002712250454351306, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00272, "step": 196, "tokens/total": 6388496, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 89812} +{"epoch": 0.76953125, "grad_norm": 0.05605921521782875, "learning_rate": 7.528718610173511e-05, "loss": 0.000789396814070642, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00079, "step": 197, "tokens/total": 6420600, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 90252} +{"epoch": 0.7734375, "grad_norm": 0.8928170800209045, "learning_rate": 7.502764873523431e-05, "loss": 0.02411472424864769, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.02441, "step": 198, "tokens/total": 6454616, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 90718} +{"epoch": 0.77734375, "grad_norm": 0.12989474833011627, "learning_rate": 7.476727793652011e-05, "loss": 0.0022076801396906376, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00221, "step": 199, "tokens/total": 6485680, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 91194} +{"epoch": 0.78125, "grad_norm": 0.2827147841453552, "learning_rate": 7.450608454068415e-05, "loss": 0.0028994400054216385, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.0029, "step": 200, "tokens/total": 6518512, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 91649} +{"epoch": 0.78515625, "grad_norm": 0.2173275202512741, "learning_rate": 7.424407941704987e-05, "loss": 0.009500865824520588, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.00955, "step": 201, "tokens/total": 6548728, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 92078} +{"epoch": 0.7890625, "grad_norm": 1.700136423110962, "learning_rate": 7.398127346871986e-05, "loss": 0.02144213207066059, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.02167, "step": 202, "tokens/total": 6582192, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 92536} +{"epoch": 0.79296875, "grad_norm": 0.15783295035362244, "learning_rate": 7.371767763212238e-05, "loss": 0.0018349229358136654, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.00184, "step": 203, "tokens/total": 6617392, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 93032} +{"epoch": 0.796875, "grad_norm": 0.015608280897140503, "learning_rate": 7.345330287655617e-05, "loss": 0.0002922064159065485, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00029, "step": 204, "tokens/total": 6648904, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 93504} +{"epoch": 0.80078125, "grad_norm": 0.1850471794605255, "learning_rate": 7.31881602037339e-05, "loss": 0.0021492016967386007, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00215, "step": 205, "tokens/total": 6679832, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 93925} +{"epoch": 0.8046875, "grad_norm": 0.7267221808433533, "learning_rate": 7.29222606473245e-05, "loss": 0.006901453714817762, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00693, "step": 206, "tokens/total": 6710784, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 94365} +{"epoch": 0.80859375, "grad_norm": 0.16473372280597687, "learning_rate": 7.265561527249383e-05, "loss": 0.0017153035150840878, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00172, "step": 207, "tokens/total": 6744848, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 94845} +{"epoch": 0.8125, "grad_norm": 0.10188228636980057, "learning_rate": 7.238823517544436e-05, "loss": 0.0017196189146488905, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00172, "step": 208, "tokens/total": 6777600, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 95285} +{"epoch": 0.81640625, "grad_norm": 0.40456584095954895, "learning_rate": 7.212013148295333e-05, "loss": 0.004568344913423061, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00458, "step": 209, "tokens/total": 6811024, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 95758} +{"epoch": 0.8203125, "grad_norm": 0.29500317573547363, "learning_rate": 7.185131535190975e-05, "loss": 0.003409844823181629, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.15, "memory/max_allocated (GiB)": 66.15, "ppl": 1.00342, "step": 210, "tokens/total": 6845376, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 96192} +{"epoch": 0.82421875, "grad_norm": 0.7075493335723877, "learning_rate": 7.158179796885005e-05, "loss": 0.019192345440387726, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01938, "step": 211, "tokens/total": 6880776, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 96641} +{"epoch": 0.828125, "grad_norm": 0.6605536937713623, "learning_rate": 7.131159054949273e-05, "loss": 0.00794963352382183, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00798, "step": 212, "tokens/total": 6916160, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 97160} +{"epoch": 0.83203125, "grad_norm": 0.35773470997810364, "learning_rate": 7.104070433827139e-05, "loss": 0.007387167774140835, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.54, "memory/max_allocated (GiB)": 63.54, "ppl": 1.00741, "step": 213, "tokens/total": 6945320, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 97611} +{"epoch": 0.8359375, "grad_norm": 0.1570860743522644, "learning_rate": 7.076915060786705e-05, "loss": 0.002963744103908539, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00297, "step": 214, "tokens/total": 6977856, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 98111} +{"epoch": 0.83984375, "grad_norm": 0.1807861477136612, "learning_rate": 7.049694065873882e-05, "loss": 0.0029980081599205732, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.003, "step": 215, "tokens/total": 7011096, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 98575} +{"epoch": 0.84375, "grad_norm": 0.4871666133403778, "learning_rate": 7.022408581865382e-05, "loss": 0.00453044893220067, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.93, "memory/max_allocated (GiB)": 63.93, "ppl": 1.00454, "step": 216, "tokens/total": 7041408, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 98991} +{"epoch": 0.84765625, "grad_norm": 0.06615452468395233, "learning_rate": 6.99505974422157e-05, "loss": 0.0007906088139861822, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.6, "memory/max_allocated (GiB)": 63.6, "ppl": 1.00079, "step": 217, "tokens/total": 7071560, "tokens/train_per_sec_per_gpu": 11.26, "tokens/trainable": 99454} +{"epoch": 0.8515625, "grad_norm": 0.01007695309817791, "learning_rate": 6.967648691039213e-05, "loss": 0.00012474186951294541, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00012, "step": 218, "tokens/total": 7104696, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 99925} +{"epoch": 0.85546875, "grad_norm": 0.07182668894529343, "learning_rate": 6.940176563004123e-05, "loss": 0.0009844396263360977, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00098, "step": 219, "tokens/total": 7138504, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 100371} +{"epoch": 0.859375, "grad_norm": 0.1120913177728653, "learning_rate": 6.912644503343682e-05, "loss": 0.001755829667672515, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00176, "step": 220, "tokens/total": 7170992, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 100855} +{"epoch": 0.86328125, "grad_norm": 0.37162479758262634, "learning_rate": 6.885053657779273e-05, "loss": 0.002598271006718278, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.0026, "step": 221, "tokens/total": 7201176, "tokens/train_per_sec_per_gpu": 6.75, "tokens/trainable": 101253} +{"epoch": 0.8671875, "grad_norm": 0.1923282891511917, "learning_rate": 6.857405174478604e-05, "loss": 0.0025343559682369232, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00254, "step": 222, "tokens/total": 7233528, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 101706} +{"epoch": 0.87109375, "grad_norm": 0.4547644555568695, "learning_rate": 6.82970020400792e-05, "loss": 0.013775174506008625, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01387, "step": 223, "tokens/total": 7267672, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 102158} +{"epoch": 0.875, "grad_norm": 0.6321426033973694, "learning_rate": 6.801939899284132e-05, "loss": 0.005456832703202963, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 62.86, "memory/max_allocated (GiB)": 62.86, "ppl": 1.00547, "step": 224, "tokens/total": 7295592, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 102580} +{"epoch": 0.87890625, "grad_norm": 1.6995550394058228, "learning_rate": 6.774125415526827e-05, "loss": 0.015882009640336037, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.01601, "step": 225, "tokens/total": 7325816, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 103063} +{"epoch": 0.8828125, "grad_norm": 0.365264892578125, "learning_rate": 6.746257910210214e-05, "loss": 0.004104663617908955, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00411, "step": 226, "tokens/total": 7360888, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 103510} +{"epoch": 0.88671875, "grad_norm": 0.12676295638084412, "learning_rate": 6.718338543014937e-05, "loss": 0.0011136239627376199, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00111, "step": 227, "tokens/total": 7390416, "tokens/train_per_sec_per_gpu": 10.05, "tokens/trainable": 103980} +{"epoch": 0.890625, "grad_norm": 0.04386964812874794, "learning_rate": 6.69036847577983e-05, "loss": 0.0004630305338650942, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00046, "step": 228, "tokens/total": 7420480, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 104413} +{"epoch": 0.89453125, "grad_norm": 0.24165570735931396, "learning_rate": 6.662348872453553e-05, "loss": 0.002723721321672201, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00273, "step": 229, "tokens/total": 7448992, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 104845} +{"epoch": 0.8984375, "grad_norm": 0.5571973323822021, "learning_rate": 6.63428089904618e-05, "loss": 0.0029496802017092705, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00295, "step": 230, "tokens/total": 7481744, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 105289} +{"epoch": 0.90234375, "grad_norm": 0.3646911382675171, "learning_rate": 6.60616572358065e-05, "loss": 0.014682111330330372, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01479, "step": 231, "tokens/total": 7516704, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 105722} +{"epoch": 0.90625, "grad_norm": 0.2986753284931183, "learning_rate": 6.578004516044172e-05, "loss": 0.00792737491428852, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.19, "memory/max_allocated (GiB)": 64.19, "ppl": 1.00796, "step": 232, "tokens/total": 7548592, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 106213} +{"epoch": 0.91015625, "grad_norm": 0.639011561870575, "learning_rate": 6.549798448339548e-05, "loss": 0.021978814154863358, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.02222, "step": 233, "tokens/total": 7579288, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 106643} +{"epoch": 0.9140625, "grad_norm": 0.2470310479402542, "learning_rate": 6.521548694236384e-05, "loss": 0.005206338129937649, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00522, "step": 234, "tokens/total": 7611672, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 107136} +{"epoch": 0.91796875, "grad_norm": 1.5670242309570312, "learning_rate": 6.493256429322259e-05, "loss": 0.005403196904808283, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00542, "step": 235, "tokens/total": 7646288, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 107588} +{"epoch": 0.921875, "grad_norm": 0.22836089134216309, "learning_rate": 6.464922830953799e-05, "loss": 0.005032885819673538, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00505, "step": 236, "tokens/total": 7680136, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 108087} +{"epoch": 0.92578125, "grad_norm": 0.12456106394529343, "learning_rate": 6.436549078207688e-05, "loss": 0.0013228687457740307, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.61, "memory/max_allocated (GiB)": 64.61, "ppl": 1.00132, "step": 237, "tokens/total": 7713072, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 108533} +{"epoch": 0.9296875, "grad_norm": 0.12802985310554504, "learning_rate": 6.408136351831592e-05, "loss": 0.0017290185205638409, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00173, "step": 238, "tokens/total": 7747856, "tokens/train_per_sec_per_gpu": 6.87, "tokens/trainable": 108965} +{"epoch": 0.93359375, "grad_norm": 0.22483210265636444, "learning_rate": 6.379685834195036e-05, "loss": 0.006162829231470823, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00618, "step": 239, "tokens/total": 7780136, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 109416} +{"epoch": 0.9375, "grad_norm": 0.07171548157930374, "learning_rate": 6.351198709240186e-05, "loss": 0.0012486852938309312, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.78, "memory/max_allocated (GiB)": 65.78, "ppl": 1.00125, "step": 240, "tokens/total": 7816344, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 109864} +{"epoch": 0.94140625, "grad_norm": 0.2607346475124359, "learning_rate": 6.32267616243259e-05, "loss": 0.0032172640785574913, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00322, "step": 241, "tokens/total": 7850408, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 110345} +{"epoch": 0.9453125, "grad_norm": 0.03972936421632767, "learning_rate": 6.294119380711849e-05, "loss": 0.0005744601949118078, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00057, "step": 242, "tokens/total": 7881528, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 110794} +{"epoch": 0.94921875, "grad_norm": 0.06568095833063126, "learning_rate": 6.265529552442209e-05, "loss": 0.0006239906069822609, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00062, "step": 243, "tokens/total": 7915712, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 111250} +{"epoch": 0.953125, "grad_norm": 0.46013128757476807, "learning_rate": 6.236907867363127e-05, "loss": 0.011512024328112602, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.61, "memory/max_allocated (GiB)": 64.61, "ppl": 1.01158, "step": 244, "tokens/total": 7947408, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 111693} +{"epoch": 0.95703125, "grad_norm": 0.20866484940052032, "learning_rate": 6.208255516539749e-05, "loss": 0.004247748292982578, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.26, "memory/max_allocated (GiB)": 65.26, "ppl": 1.00426, "step": 245, "tokens/total": 7979600, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 112158} +{"epoch": 0.9609375, "grad_norm": 0.1305297464132309, "learning_rate": 6.179573692313344e-05, "loss": 0.0026735998690128326, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00268, "step": 246, "tokens/total": 8010848, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 112601} +{"epoch": 0.96484375, "grad_norm": 0.1561560183763504, "learning_rate": 6.150863588251694e-05, "loss": 0.0014340992784127593, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.00144, "step": 247, "tokens/total": 8040632, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 113016} +{"epoch": 0.96875, "grad_norm": 0.03442670777440071, "learning_rate": 6.122126399099419e-05, "loss": 0.0003439930151216686, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00034, "step": 248, "tokens/total": 8072464, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 113476} +{"epoch": 0.97265625, "grad_norm": 0.4546850323677063, "learning_rate": 6.0933633207282615e-05, "loss": 0.005530410911887884, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.96, "memory/max_allocated (GiB)": 64.96, "ppl": 1.00555, "step": 249, "tokens/total": 8106568, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 113941} +{"epoch": 0.9765625, "grad_norm": 0.17342089116573334, "learning_rate": 6.064575550087316e-05, "loss": 0.0023081672843545675, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00231, "step": 250, "tokens/total": 8138496, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 114434} +{"epoch": 0.98046875, "grad_norm": 0.016219334676861763, "learning_rate": 6.0357642851532245e-05, "loss": 0.00015776742657180876, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00016, "step": 251, "tokens/total": 8168080, "tokens/train_per_sec_per_gpu": 10.17, "tokens/trainable": 114898} +{"epoch": 0.984375, "grad_norm": 0.013694246299564838, "learning_rate": 6.0069307248803294e-05, "loss": 8.17549298517406e-05, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00008, "step": 252, "tokens/total": 8200888, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 115331} +{"epoch": 0.98828125, "grad_norm": 0.18211525678634644, "learning_rate": 5.9780760691507635e-05, "loss": 0.001090601203031838, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00109, "step": 253, "tokens/total": 8230704, "tokens/train_per_sec_per_gpu": 9.64, "tokens/trainable": 115732} +{"epoch": 0.9921875, "grad_norm": 0.20055967569351196, "learning_rate": 5.9492015187245334e-05, "loss": 0.003772688563913107, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00378, "step": 254, "tokens/total": 8266560, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 116221} +{"epoch": 0.99609375, "grad_norm": 0.2085074782371521, "learning_rate": 5.920308275189541e-05, "loss": 0.0030748583376407623, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.19, "memory/max_allocated (GiB)": 65.19, "ppl": 1.00308, "step": 255, "tokens/total": 8300184, "tokens/train_per_sec_per_gpu": 7.02, "tokens/trainable": 116684} +{"epoch": 1.0, "grad_norm": 0.004859834909439087, "learning_rate": 5.8913975409115874e-05, "loss": 2.354940806981176e-05, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00002, "step": 256, "tokens/total": 8333624, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 117126} +{"epoch": 1.00390625, "grad_norm": 0.012713676318526268, "learning_rate": 5.8624705189843395e-05, "loss": 6.401107384590432e-05, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00006, "step": 257, "tokens/total": 8364976, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 117592} +{"epoch": 1.0078125, "grad_norm": 0.005823029670864344, "learning_rate": 5.833528413179249e-05, "loss": 4.923506639897823e-05, "memory/device_reserved (GiB)": 66.62, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00005, "step": 258, "tokens/total": 8399568, "tokens/train_per_sec_per_gpu": 6.94, "tokens/trainable": 118030} +{"epoch": 1.01171875, "grad_norm": 0.8352934122085571, "learning_rate": 5.80457242789548e-05, "loss": 0.0031613868195563555, "memory/device_reserved (GiB)": 66.62, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00317, "step": 259, "tokens/total": 8430616, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 118470} +{"epoch": 1.015625, "grad_norm": 0.1852319985628128, "learning_rate": 5.77560376810977e-05, "loss": 0.002247996861115098, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00225, "step": 260, "tokens/total": 8462696, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 118935} +{"epoch": 1.01953125, "grad_norm": 0.24912264943122864, "learning_rate": 5.7466236393263005e-05, "loss": 0.0009054704569280148, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00091, "step": 261, "tokens/total": 8496152, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 119381} +{"epoch": 1.0234375, "grad_norm": 0.1327054649591446, "learning_rate": 5.717633247526522e-05, "loss": 0.0007402639603242278, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00074, "step": 262, "tokens/total": 8532912, "tokens/train_per_sec_per_gpu": 7.02, "tokens/trainable": 119849} +{"epoch": 1.02734375, "grad_norm": 0.004797554574906826, "learning_rate": 5.688633799118971e-05, "loss": 3.785236913245171e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.00004, "step": 263, "tokens/total": 8565376, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 120319} +{"epoch": 1.03125, "grad_norm": 0.005811900831758976, "learning_rate": 5.659626500889066e-05, "loss": 3.8053800381021574e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00004, "step": 264, "tokens/total": 8597640, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 120804} +{"epoch": 1.03515625, "grad_norm": 0.002337844343855977, "learning_rate": 5.6306125599488905e-05, "loss": 2.0687344658654183e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.00002, "step": 265, "tokens/total": 8627312, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 121233} +{"epoch": 1.0390625, "grad_norm": 0.038318052887916565, "learning_rate": 5.601593183686955e-05, "loss": 0.00018003251170739532, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.87, "memory/max_allocated (GiB)": 64.87, "ppl": 1.00018, "step": 266, "tokens/total": 8659208, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 121671} +{"epoch": 1.04296875, "grad_norm": 0.03242640942335129, "learning_rate": 5.572569579717961e-05, "loss": 8.040816464927047e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.00008, "step": 267, "tokens/total": 8693088, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 122141} +{"epoch": 1.046875, "grad_norm": 0.06524225324392319, "learning_rate": 5.543542955832538e-05, "loss": 0.0002263579808641225, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00023, "step": 268, "tokens/total": 8725576, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 122556} +{"epoch": 1.05078125, "grad_norm": 0.4352176785469055, "learning_rate": 5.514514519946986e-05, "loss": 0.002573086181655526, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00258, "step": 269, "tokens/total": 8755192, "tokens/train_per_sec_per_gpu": 10.25, "tokens/trainable": 122988} +{"epoch": 1.0546875, "grad_norm": 0.00019334634998813272, "learning_rate": 5.485485480053015e-05, "loss": 3.9852138797868975e-06, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.0, "step": 270, "tokens/total": 8787704, "tokens/train_per_sec_per_gpu": 10.47, "tokens/trainable": 123478} +{"epoch": 1.05859375, "grad_norm": 0.010965397581458092, "learning_rate": 5.4564570441674645e-05, "loss": 6.391105853253976e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00006, "step": 271, "tokens/total": 8819768, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 123932} +{"epoch": 1.0625, "grad_norm": 1.130143165588379, "learning_rate": 5.42743042028204e-05, "loss": 0.003069486701861024, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00307, "step": 272, "tokens/total": 8853352, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 124428} +{"epoch": 1.06640625, "grad_norm": 0.1574181467294693, "learning_rate": 5.3984068163130464e-05, "loss": 0.00087478052591905, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00088, "step": 273, "tokens/total": 8886840, "tokens/train_per_sec_per_gpu": 7.12, "tokens/trainable": 124866} +{"epoch": 1.0703125, "grad_norm": 0.7642207741737366, "learning_rate": 5.369387440051111e-05, "loss": 0.004208532627671957, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00422, "step": 274, "tokens/total": 8920808, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 125324} +{"epoch": 1.07421875, "grad_norm": 0.7824990153312683, "learning_rate": 5.340373499110935e-05, "loss": 0.010431399568915367, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.01049, "step": 275, "tokens/total": 8955880, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 125813} +{"epoch": 1.078125, "grad_norm": 0.003488674759864807, "learning_rate": 5.3113662008810304e-05, "loss": 3.247799031669274e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.00003, "step": 276, "tokens/total": 8989520, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 126252} +{"epoch": 1.08203125, "grad_norm": 0.13161823153495789, "learning_rate": 5.282366752473479e-05, "loss": 0.000497160479426384, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.0005, "step": 277, "tokens/total": 9024680, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 126696} +{"epoch": 1.0859375, "grad_norm": 0.38812434673309326, "learning_rate": 5.2533763606737005e-05, "loss": 0.0034632482565939426, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00347, "step": 278, "tokens/total": 9056208, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 127173} +{"epoch": 1.08984375, "grad_norm": 0.00692913681268692, "learning_rate": 5.224396231890232e-05, "loss": 2.724806108744815e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00003, "step": 279, "tokens/total": 9086376, "tokens/train_per_sec_per_gpu": 9.88, "tokens/trainable": 127637} +{"epoch": 1.09375, "grad_norm": 0.2628335654735565, "learning_rate": 5.195427572104522e-05, "loss": 0.0030731195583939552, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00308, "step": 280, "tokens/total": 9118944, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 128112} +{"epoch": 1.09765625, "grad_norm": 0.014181901700794697, "learning_rate": 5.166471586820751e-05, "loss": 4.373253614176065e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00004, "step": 281, "tokens/total": 9152920, "tokens/train_per_sec_per_gpu": 6.98, "tokens/trainable": 128584} +{"epoch": 1.1015625, "grad_norm": 0.4398467242717743, "learning_rate": 5.1375294810156615e-05, "loss": 0.006146387197077274, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00617, "step": 282, "tokens/total": 9185728, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 129051} +{"epoch": 1.10546875, "grad_norm": 0.0011290027759969234, "learning_rate": 5.1086024590884144e-05, "loss": 1.2414538105076645e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00001, "step": 283, "tokens/total": 9217984, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 129494} +{"epoch": 1.109375, "grad_norm": 0.031603600829839706, "learning_rate": 5.079691724810461e-05, "loss": 0.00017892345204018056, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00018, "step": 284, "tokens/total": 9253160, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 129983} +{"epoch": 1.11328125, "grad_norm": 0.19030626118183136, "learning_rate": 5.0507984812754684e-05, "loss": 0.0008448568405583501, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00085, "step": 285, "tokens/total": 9286136, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 130405} +{"epoch": 1.1171875, "grad_norm": 0.022237690165638924, "learning_rate": 5.021923930849237e-05, "loss": 0.00021857331739738584, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00022, "step": 286, "tokens/total": 9318360, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 130829} +{"epoch": 1.12109375, "grad_norm": 0.6903882622718811, "learning_rate": 4.99306927511967e-05, "loss": 0.021556351333856583, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.02179, "step": 287, "tokens/total": 9350288, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 131287} +{"epoch": 1.125, "grad_norm": 0.03543877601623535, "learning_rate": 4.964235714846775e-05, "loss": 0.0003362484276294708, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00034, "step": 288, "tokens/total": 9382104, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 131712} +{"epoch": 1.12890625, "grad_norm": 0.25662559270858765, "learning_rate": 4.9354244499126866e-05, "loss": 0.0015159222530201077, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00152, "step": 289, "tokens/total": 9414392, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 132168} +{"epoch": 1.1328125, "grad_norm": 0.02180466800928116, "learning_rate": 4.90663667927174e-05, "loss": 0.00028495455626398325, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00028, "step": 290, "tokens/total": 9447112, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 132626} +{"epoch": 1.13671875, "grad_norm": 1.552191138267517, "learning_rate": 4.877873600900581e-05, "loss": 0.0020401489455252886, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00204, "step": 291, "tokens/total": 9483400, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 133119} +{"epoch": 1.140625, "grad_norm": 0.0019201550167053938, "learning_rate": 4.849136411748306e-05, "loss": 3.693298640428111e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.48, "memory/max_allocated (GiB)": 64.48, "ppl": 1.00004, "step": 292, "tokens/total": 9513896, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 133550} +{"epoch": 1.14453125, "grad_norm": 0.023754829540848732, "learning_rate": 4.8204263076866574e-05, "loss": 0.0002675018331501633, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00027, "step": 293, "tokens/total": 9550184, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 133992} +{"epoch": 1.1484375, "grad_norm": 0.01859096623957157, "learning_rate": 4.791744483460251e-05, "loss": 0.00019233740749768913, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00019, "step": 294, "tokens/total": 9583920, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 134467} +{"epoch": 1.15234375, "grad_norm": 0.12525470554828644, "learning_rate": 4.7630921326368736e-05, "loss": 0.0013153908075764775, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00132, "step": 295, "tokens/total": 9617288, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 134895} +{"epoch": 1.15625, "grad_norm": 0.6581651568412781, "learning_rate": 4.7344704475577916e-05, "loss": 0.01291360892355442, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 62.94, "memory/max_allocated (GiB)": 62.94, "ppl": 1.013, "step": 296, "tokens/total": 9645600, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 135321} +{"epoch": 1.16015625, "grad_norm": 0.1637813150882721, "learning_rate": 4.705880619288153e-05, "loss": 0.0011246285866945982, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00113, "step": 297, "tokens/total": 9679456, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 135765} +{"epoch": 1.1640625, "grad_norm": 0.006822012364864349, "learning_rate": 4.677323837567412e-05, "loss": 0.0001005522208288312, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.0001, "step": 298, "tokens/total": 9711696, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 136196} +{"epoch": 1.16796875, "grad_norm": 0.20008648931980133, "learning_rate": 4.6488012907598146e-05, "loss": 0.0020385899115353823, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00204, "step": 299, "tokens/total": 9745792, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 136649} +{"epoch": 1.171875, "grad_norm": 0.04121082276105881, "learning_rate": 4.620314165804964e-05, "loss": 0.0005246884538792074, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00052, "step": 300, "tokens/total": 9780648, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 137128} +{"epoch": 1.17578125, "grad_norm": 0.1850910186767578, "learning_rate": 4.591863648168407e-05, "loss": 0.0014834025641903281, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00148, "step": 301, "tokens/total": 9813032, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 137585} +{"epoch": 1.1796875, "grad_norm": 0.19230148196220398, "learning_rate": 4.5634509217923135e-05, "loss": 0.001419153530150652, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00142, "step": 302, "tokens/total": 9845200, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 138054} +{"epoch": 1.18359375, "grad_norm": 0.08793559670448303, "learning_rate": 4.535077169046201e-05, "loss": 0.0006030077929608524, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.29, "memory/max_allocated (GiB)": 65.29, "ppl": 1.0006, "step": 303, "tokens/total": 9878616, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 138559} +{"epoch": 1.1875, "grad_norm": 0.0829860270023346, "learning_rate": 4.506743570677743e-05, "loss": 0.0008395861950702965, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00084, "step": 304, "tokens/total": 9910776, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 139050} +{"epoch": 1.19140625, "grad_norm": 0.02472522296011448, "learning_rate": 4.478451305763618e-05, "loss": 0.00030416276422329247, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.0003, "step": 305, "tokens/total": 9945480, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 139525} +{"epoch": 1.1953125, "grad_norm": 0.02882031910121441, "learning_rate": 4.450201551660454e-05, "loss": 0.00030581862665712833, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.00031, "step": 306, "tokens/total": 9975992, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 139969} +{"epoch": 1.19921875, "grad_norm": 0.007290355395525694, "learning_rate": 4.4219954839558276e-05, "loss": 0.00013905370724387467, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.00014, "step": 307, "tokens/total": 10006440, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 140443} +{"epoch": 1.203125, "grad_norm": 0.004577075596898794, "learning_rate": 4.393834276419352e-05, "loss": 7.472388097085059e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00007, "step": 308, "tokens/total": 10042928, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 140926} +{"epoch": 1.20703125, "grad_norm": 0.012431034818291664, "learning_rate": 4.36571910095382e-05, "loss": 0.0001267819170607254, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00013, "step": 309, "tokens/total": 10075184, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 141344} +{"epoch": 1.2109375, "grad_norm": 0.12630753219127655, "learning_rate": 4.337651127546448e-05, "loss": 0.0006226495606824756, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00062, "step": 310, "tokens/total": 10106656, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 141753} +{"epoch": 1.21484375, "grad_norm": 0.07808469980955124, "learning_rate": 4.3096315242201736e-05, "loss": 0.0004721173318102956, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.82, "memory/max_allocated (GiB)": 63.82, "ppl": 1.00047, "step": 311, "tokens/total": 10138536, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 142219} +{"epoch": 1.21875, "grad_norm": 0.006200388073921204, "learning_rate": 4.2816614569850635e-05, "loss": 4.786123463418335e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.00005, "step": 312, "tokens/total": 10165768, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 142617} +{"epoch": 1.22265625, "grad_norm": 0.005309208761900663, "learning_rate": 4.2537420897897864e-05, "loss": 5.539559060707688e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00006, "step": 313, "tokens/total": 10201624, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 143103} +{"epoch": 1.2265625, "grad_norm": 0.0038220398128032684, "learning_rate": 4.225874584473174e-05, "loss": 5.4321328207151964e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00005, "step": 314, "tokens/total": 10233304, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 143535} +{"epoch": 1.23046875, "grad_norm": 0.0314180850982666, "learning_rate": 4.19806010071587e-05, "loss": 0.00013404338096734136, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00013, "step": 315, "tokens/total": 10267904, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 144009} +{"epoch": 1.234375, "grad_norm": 0.3356620669364929, "learning_rate": 4.170299795992081e-05, "loss": 0.0017564042937010527, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00176, "step": 316, "tokens/total": 10301280, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 144460} +{"epoch": 1.23828125, "grad_norm": 0.13748498260974884, "learning_rate": 4.142594825521398e-05, "loss": 0.0011545403394848108, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00116, "step": 317, "tokens/total": 10337072, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 144936} +{"epoch": 1.2421875, "grad_norm": 0.007199449464678764, "learning_rate": 4.114946342220728e-05, "loss": 8.12460930319503e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.69, "memory/max_allocated (GiB)": 64.69, "ppl": 1.00008, "step": 318, "tokens/total": 10368688, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 145375} +{"epoch": 1.24609375, "grad_norm": 0.14371590316295624, "learning_rate": 4.087355496656321e-05, "loss": 0.0015588192036375403, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00156, "step": 319, "tokens/total": 10400752, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 145849} +{"epoch": 1.25, "grad_norm": 0.07962356507778168, "learning_rate": 4.05982343699588e-05, "loss": 0.00039897041278891265, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.0004, "step": 320, "tokens/total": 10432112, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 146318} +{"epoch": 1.25390625, "grad_norm": 0.18296845257282257, "learning_rate": 4.0323513089607876e-05, "loss": 0.0015840253327041864, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00159, "step": 321, "tokens/total": 10465472, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 146821} +{"epoch": 1.2578125, "grad_norm": 0.005578713491559029, "learning_rate": 4.004940255778431e-05, "loss": 3.2962158002192155e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00003, "step": 322, "tokens/total": 10496336, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 147285} +{"epoch": 1.26171875, "grad_norm": 0.0018888604827225208, "learning_rate": 3.977591418134619e-05, "loss": 1.96277724171523e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00002, "step": 323, "tokens/total": 10525864, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 147729} +{"epoch": 1.265625, "grad_norm": 0.18126511573791504, "learning_rate": 3.95030593412612e-05, "loss": 0.0007941887597553432, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00079, "step": 324, "tokens/total": 10559680, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 148169} +{"epoch": 1.26953125, "grad_norm": 0.32462435960769653, "learning_rate": 3.923084939213296e-05, "loss": 0.004452345427125692, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00446, "step": 325, "tokens/total": 10595400, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 148630} +{"epoch": 1.2734375, "grad_norm": 0.0014148361515253782, "learning_rate": 3.895929566172861e-05, "loss": 2.177435817429796e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00002, "step": 326, "tokens/total": 10626680, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 149063} +{"epoch": 1.27734375, "grad_norm": 0.036054279655218124, "learning_rate": 3.868840945050728e-05, "loss": 0.00017480777751188725, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00017, "step": 327, "tokens/total": 10661784, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 149549} +{"epoch": 1.28125, "grad_norm": 0.5097113251686096, "learning_rate": 3.841820203114995e-05, "loss": 0.006486574187874794, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.31, "memory/max_allocated (GiB)": 65.31, "ppl": 1.00651, "step": 328, "tokens/total": 10696184, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 150015} +{"epoch": 1.28515625, "grad_norm": 0.007786806672811508, "learning_rate": 3.814868464809027e-05, "loss": 5.6776545534376055e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00006, "step": 329, "tokens/total": 10729376, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 150514} +{"epoch": 1.2890625, "grad_norm": 0.019388524815440178, "learning_rate": 3.787986851704667e-05, "loss": 7.852977432776242e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00008, "step": 330, "tokens/total": 10764824, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 151021} +{"epoch": 1.29296875, "grad_norm": 0.016769573092460632, "learning_rate": 3.7611764824555654e-05, "loss": 0.0001791710965335369, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00018, "step": 331, "tokens/total": 10797072, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 151486} +{"epoch": 1.296875, "grad_norm": 0.27959099411964417, "learning_rate": 3.734438472750619e-05, "loss": 0.002402051817625761, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.0024, "step": 332, "tokens/total": 10829376, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 151956} +{"epoch": 1.30078125, "grad_norm": 0.0006426776526495814, "learning_rate": 3.707773935267552e-05, "loss": 1.280333071917994e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00001, "step": 333, "tokens/total": 10863512, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 152442} +{"epoch": 1.3046875, "grad_norm": 0.0019409045344218612, "learning_rate": 3.68118397962661e-05, "loss": 2.831396159308497e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00003, "step": 334, "tokens/total": 10894432, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 152890} +{"epoch": 1.30859375, "grad_norm": 0.47676903009414673, "learning_rate": 3.654669712344384e-05, "loss": 0.006155196111649275, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.34, "memory/max_allocated (GiB)": 63.34, "ppl": 1.00617, "step": 335, "tokens/total": 10924848, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 153368} +{"epoch": 1.3125, "grad_norm": 0.009906524792313576, "learning_rate": 3.628232236787763e-05, "loss": 4.6675209887325764e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00005, "step": 336, "tokens/total": 10959024, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 153809} +{"epoch": 1.31640625, "grad_norm": 0.3936930000782013, "learning_rate": 3.6018726531280144e-05, "loss": 0.0035286180209368467, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00353, "step": 337, "tokens/total": 10991088, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 154258} +{"epoch": 1.3203125, "grad_norm": 0.8746547698974609, "learning_rate": 3.575592058295017e-05, "loss": 0.004444637335836887, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00445, "step": 338, "tokens/total": 11023904, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 154704} +{"epoch": 1.32421875, "grad_norm": 0.06444619596004486, "learning_rate": 3.549391545931585e-05, "loss": 0.00014640983135905117, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00015, "step": 339, "tokens/total": 11057680, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 155157} +{"epoch": 1.328125, "grad_norm": 0.11422253400087357, "learning_rate": 3.5232722063479914e-05, "loss": 0.0003720279491972178, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00037, "step": 340, "tokens/total": 11089800, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 155582} +{"epoch": 1.33203125, "grad_norm": 0.0015656936448067427, "learning_rate": 3.49723512647657e-05, "loss": 1.0703734915296081e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00001, "step": 341, "tokens/total": 11117960, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 156047} +{"epoch": 1.3359375, "grad_norm": 0.2380523830652237, "learning_rate": 3.471281389826491e-05, "loss": 0.0009368701139464974, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00094, "step": 342, "tokens/total": 11151888, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 156500} +{"epoch": 1.33984375, "grad_norm": 0.06183967739343643, "learning_rate": 3.4454120764386764e-05, "loss": 0.00035785080399364233, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.22, "memory/max_allocated (GiB)": 65.22, "ppl": 1.00036, "step": 343, "tokens/total": 11184568, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 156931} +{"epoch": 1.34375, "grad_norm": 0.4181990325450897, "learning_rate": 3.4196282628408526e-05, "loss": 0.020239345729351044, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.02045, "step": 344, "tokens/total": 11216400, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 157385} +{"epoch": 1.34765625, "grad_norm": 0.016466066241264343, "learning_rate": 3.3939310220027456e-05, "loss": 0.0001240487035829574, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00012, "step": 345, "tokens/total": 11250768, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 157817} +{"epoch": 1.3515625, "grad_norm": 0.007648918312042952, "learning_rate": 3.3683214232914404e-05, "loss": 6.281316018430516e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00006, "step": 346, "tokens/total": 11281808, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 158273} +{"epoch": 1.35546875, "grad_norm": 0.011521294713020325, "learning_rate": 3.342800532426873e-05, "loss": 0.00012367793533485383, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00012, "step": 347, "tokens/total": 11313968, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 158728} +{"epoch": 1.359375, "grad_norm": 0.0009286186541430652, "learning_rate": 3.317369411437484e-05, "loss": 1.288914427277632e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00001, "step": 348, "tokens/total": 11347288, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 159168} +{"epoch": 1.36328125, "grad_norm": 0.8967955708503723, "learning_rate": 3.292029118616024e-05, "loss": 0.006555600557476282, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00658, "step": 349, "tokens/total": 11377128, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 159592} +{"epoch": 1.3671875, "grad_norm": 0.006102315615862608, "learning_rate": 3.266780708475511e-05, "loss": 9.069789666682482e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00009, "step": 350, "tokens/total": 11409488, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 160093} +{"epoch": 1.37109375, "grad_norm": 0.1942141056060791, "learning_rate": 3.241625231705354e-05, "loss": 0.0009175800369121134, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00092, "step": 351, "tokens/total": 11442040, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 160534} +{"epoch": 1.375, "grad_norm": 0.8588467240333557, "learning_rate": 3.216563735127618e-05, "loss": 0.006315165665000677, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00634, "step": 352, "tokens/total": 11478312, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 161009} +{"epoch": 1.37890625, "grad_norm": 1.248298168182373, "learning_rate": 3.191597261653475e-05, "loss": 0.004352542571723461, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.93, "memory/max_allocated (GiB)": 63.93, "ppl": 1.00436, "step": 353, "tokens/total": 11509008, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 161450} +{"epoch": 1.3828125, "grad_norm": 0.43369340896606445, "learning_rate": 3.166726850239794e-05, "loss": 0.00816345028579235, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.0082, "step": 354, "tokens/total": 11541264, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 161900} +{"epoch": 1.38671875, "grad_norm": 0.0038864680100232363, "learning_rate": 3.141953535845912e-05, "loss": 4.6782246499788016e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00005, "step": 355, "tokens/total": 11573072, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 162358} +{"epoch": 1.390625, "grad_norm": 0.012187984772026539, "learning_rate": 3.11727834939056e-05, "loss": 0.00012193253496661782, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00012, "step": 356, "tokens/total": 11605224, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 162845} +{"epoch": 1.39453125, "grad_norm": 0.3167530596256256, "learning_rate": 3.092702317708967e-05, "loss": 0.005443981382995844, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00546, "step": 357, "tokens/total": 11637512, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 163299} +{"epoch": 1.3984375, "grad_norm": 0.08372802287340164, "learning_rate": 3.0682264635101276e-05, "loss": 0.0008326321840286255, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00083, "step": 358, "tokens/total": 11671280, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 163774} +{"epoch": 1.40234375, "grad_norm": 0.020801128819584846, "learning_rate": 3.0438518053342407e-05, "loss": 0.0002660672180354595, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00027, "step": 359, "tokens/total": 11702008, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 164199} +{"epoch": 1.40625, "grad_norm": 0.144286647439003, "learning_rate": 3.0195793575103266e-05, "loss": 0.0018363583367317915, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00184, "step": 360, "tokens/total": 11732032, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 164667} +{"epoch": 1.41015625, "grad_norm": 0.024546006694436073, "learning_rate": 2.9954101301140146e-05, "loss": 0.00035448651760816574, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00035, "step": 361, "tokens/total": 11767472, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.048251718282699585, "learning_rate": 2.9713451289255123e-05, "loss": 0.0005163613823242486, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00052, "step": 362, "tokens/total": 11798432, "tokens/train_per_sec_per_gpu": 10.4, "tokens/trainable": 165575} +{"epoch": 1.41796875, "grad_norm": 0.1366114467382431, "learning_rate": 2.9473853553877484e-05, "loss": 0.0013148970901966095, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00132, "step": 363, "tokens/total": 11833144, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 166060} +{"epoch": 1.421875, "grad_norm": 0.026221897453069687, "learning_rate": 2.9235318065647e-05, "loss": 0.00024692274746485054, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.00025, "step": 364, "tokens/total": 11865856, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 166478} +{"epoch": 1.42578125, "grad_norm": 0.012094513513147831, "learning_rate": 2.8997854750998964e-05, "loss": 0.0001573253539390862, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00016, "step": 365, "tokens/total": 11900400, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 166941} +{"epoch": 1.4296875, "grad_norm": 0.011677906848490238, "learning_rate": 2.8761473491751258e-05, "loss": 0.00015704153338447213, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.49, "memory/max_allocated (GiB)": 63.49, "ppl": 1.00016, "step": 366, "tokens/total": 11931368, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 167381} +{"epoch": 1.43359375, "grad_norm": 0.27864211797714233, "learning_rate": 2.8526184124692883e-05, "loss": 0.002993653528392315, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.003, "step": 367, "tokens/total": 11963320, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 167830} +{"epoch": 1.4375, "grad_norm": 0.11062923073768616, "learning_rate": 2.829199644117484e-05, "loss": 0.0005098604597151279, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00051, "step": 368, "tokens/total": 11997536, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 168266} +{"epoch": 1.44140625, "grad_norm": 0.21281157433986664, "learning_rate": 2.8058920186702553e-05, "loss": 0.002410895423963666, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00241, "step": 369, "tokens/total": 12029528, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 168716} +{"epoch": 1.4453125, "grad_norm": 0.11342816054821014, "learning_rate": 2.782696506053033e-05, "loss": 0.0007816141587682068, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00078, "step": 370, "tokens/total": 12062504, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 169176} +{"epoch": 1.44921875, "grad_norm": 0.10491232573986053, "learning_rate": 2.7596140715257824e-05, "loss": 0.001015838934108615, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.78, "memory/max_allocated (GiB)": 65.78, "ppl": 1.00102, "step": 371, "tokens/total": 12097376, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 169591} +{"epoch": 1.453125, "grad_norm": 0.02595425583422184, "learning_rate": 2.7366456756428184e-05, "loss": 0.00025674476637504995, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00026, "step": 372, "tokens/total": 12131920, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 170041} +{"epoch": 1.45703125, "grad_norm": 0.1259227693080902, "learning_rate": 2.7137922742128486e-05, "loss": 0.001352685852907598, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00135, "step": 373, "tokens/total": 12164528, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 170516} +{"epoch": 1.4609375, "grad_norm": 0.02950296737253666, "learning_rate": 2.691054818259188e-05, "loss": 0.00023557007079944015, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.00024, "step": 374, "tokens/total": 12197376, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 170952} +{"epoch": 1.46484375, "grad_norm": 0.4985104203224182, "learning_rate": 2.6684342539801933e-05, "loss": 0.0030715109314769506, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00308, "step": 375, "tokens/total": 12233104, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 171424} +{"epoch": 1.46875, "grad_norm": 0.014400103129446507, "learning_rate": 2.645931522709877e-05, "loss": 0.000137602532049641, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.69, "memory/max_allocated (GiB)": 65.69, "ppl": 1.00014, "step": 376, "tokens/total": 12269224, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 171884} +{"epoch": 1.47265625, "grad_norm": 0.0070048789493739605, "learning_rate": 2.6235475608787365e-05, "loss": 5.303946090862155e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00005, "step": 377, "tokens/total": 12302952, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 172404} +{"epoch": 1.4765625, "grad_norm": 0.014220093376934528, "learning_rate": 2.6012832999747916e-05, "loss": 0.00013805269554723054, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00014, "step": 378, "tokens/total": 12338424, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 172871} +{"epoch": 1.48046875, "grad_norm": 0.013668975792825222, "learning_rate": 2.579139666504821e-05, "loss": 0.00013318519631866366, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00013, "step": 379, "tokens/total": 12369144, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 173329} +{"epoch": 1.484375, "grad_norm": 0.012582716532051563, "learning_rate": 2.557117581955798e-05, "loss": 0.00010486728569958359, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.0001, "step": 380, "tokens/total": 12403352, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 173827} +{"epoch": 1.48828125, "grad_norm": 0.07097526639699936, "learning_rate": 2.5352179627565532e-05, "loss": 0.0005789579590782523, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00058, "step": 381, "tokens/total": 12441056, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 174287} +{"epoch": 1.4921875, "grad_norm": 0.248711958527565, "learning_rate": 2.5134417202396277e-05, "loss": 0.0016068138647824526, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.00161, "step": 382, "tokens/total": 12474080, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 174731} +{"epoch": 1.49609375, "grad_norm": 0.06212310492992401, "learning_rate": 2.491789760603361e-05, "loss": 0.0005159197608008981, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00052, "step": 383, "tokens/total": 12508992, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 175183} +{"epoch": 1.5, "grad_norm": 0.10799785703420639, "learning_rate": 2.4702629848741764e-05, "loss": 0.0007174736820161343, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00072, "step": 384, "tokens/total": 12540896, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 175655} +{"epoch": 1.50390625, "grad_norm": 0.08180869370698929, "learning_rate": 2.4488622888690785e-05, "loss": 0.0008082831045612693, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00081, "step": 385, "tokens/total": 12568416, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 176085} +{"epoch": 1.5078125, "grad_norm": 0.005075998138636351, "learning_rate": 2.427588563158384e-05, "loss": 5.4831180023029447e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00005, "step": 386, "tokens/total": 12599792, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 176549} +{"epoch": 1.51171875, "grad_norm": 0.010015618987381458, "learning_rate": 2.406442693028651e-05, "loss": 8.394342148676515e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00008, "step": 387, "tokens/total": 12631224, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 176995} +{"epoch": 1.515625, "grad_norm": 0.004169133026152849, "learning_rate": 2.3854255584458547e-05, "loss": 2.9372680728556588e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00003, "step": 388, "tokens/total": 12667240, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 177468} +{"epoch": 1.51953125, "grad_norm": 0.19618503749370575, "learning_rate": 2.3645380340187508e-05, "loss": 0.0014775160234421492, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.54, "memory/max_allocated (GiB)": 65.54, "ppl": 1.00148, "step": 389, "tokens/total": 12700488, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 177961} +{"epoch": 1.5234375, "grad_norm": 0.5926028490066528, "learning_rate": 2.3437809889624914e-05, "loss": 0.0022107521072030067, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.00221, "step": 390, "tokens/total": 12734528, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 178417} +{"epoch": 1.52734375, "grad_norm": 0.031491734087467194, "learning_rate": 2.3231552870624487e-05, "loss": 0.0001703481248114258, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00017, "step": 391, "tokens/total": 12767152, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 178862} +{"epoch": 1.53125, "grad_norm": 0.6953549385070801, "learning_rate": 2.3026617866382657e-05, "loss": 0.003110898192971945, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.00312, "step": 392, "tokens/total": 12798168, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 179320} +{"epoch": 1.53515625, "grad_norm": 0.010607339441776276, "learning_rate": 2.2823013405081507e-05, "loss": 7.893462316133082e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00008, "step": 393, "tokens/total": 12828896, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 179758} +{"epoch": 1.5390625, "grad_norm": 0.09709280729293823, "learning_rate": 2.2620747959533722e-05, "loss": 0.0008588555501773953, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00086, "step": 394, "tokens/total": 12860024, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 180211} +{"epoch": 1.54296875, "grad_norm": 0.5800950527191162, "learning_rate": 2.2419829946830123e-05, "loss": 0.0034307290334254503, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00344, "step": 395, "tokens/total": 12891472, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 180637} +{"epoch": 1.546875, "grad_norm": 0.004511674400418997, "learning_rate": 2.2220267727989325e-05, "loss": 3.947218647226691e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00004, "step": 396, "tokens/total": 12925648, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 181142} +{"epoch": 1.55078125, "grad_norm": 0.010950482450425625, "learning_rate": 2.202206960760984e-05, "loss": 7.383717456832528e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.77, "memory/max_allocated (GiB)": 64.77, "ppl": 1.00007, "step": 397, "tokens/total": 12959200, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 181597} +{"epoch": 1.5546875, "grad_norm": 0.09190995991230011, "learning_rate": 2.182524383352446e-05, "loss": 0.0005258393357507885, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00053, "step": 398, "tokens/total": 12989624, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 182030} +{"epoch": 1.55859375, "grad_norm": 0.0007179180392995477, "learning_rate": 2.1629798596457056e-05, "loss": 1.6045578377088532e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 62.97, "memory/max_allocated (GiB)": 62.97, "ppl": 1.00002, "step": 399, "tokens/total": 13019040, "tokens/train_per_sec_per_gpu": 10.24, "tokens/trainable": 182496} +{"epoch": 1.5625, "grad_norm": 0.14119397103786469, "learning_rate": 2.1435742029681725e-05, "loss": 0.0008283478091470897, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00083, "step": 400, "tokens/total": 13051808, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 182963} +{"epoch": 1.56640625, "grad_norm": 0.5273993015289307, "learning_rate": 2.124308220868431e-05, "loss": 0.0004142375837545842, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00041, "step": 401, "tokens/total": 13085520, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 183416} +{"epoch": 1.5703125, "grad_norm": 0.00314325257204473, "learning_rate": 2.105182715082638e-05, "loss": 4.025848465971649e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00004, "step": 402, "tokens/total": 13116224, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 183857} +{"epoch": 1.57421875, "grad_norm": 0.026588037610054016, "learning_rate": 2.0861984815011552e-05, "loss": 0.0001263161248061806, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00013, "step": 403, "tokens/total": 13149984, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 184296} +{"epoch": 1.578125, "grad_norm": 0.15686674416065216, "learning_rate": 2.0673563101354323e-05, "loss": 0.0009332336485385895, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00093, "step": 404, "tokens/total": 13185488, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 184719} +{"epoch": 1.58203125, "grad_norm": 12.026152610778809, "learning_rate": 2.0486569850851317e-05, "loss": 0.00031929323449730873, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00032, "step": 405, "tokens/total": 13216320, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 185172} +{"epoch": 1.5859375, "grad_norm": 0.010833998210728168, "learning_rate": 2.0301012845054956e-05, "loss": 5.2690189477289096e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00005, "step": 406, "tokens/total": 13248136, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 185646} +{"epoch": 1.58984375, "grad_norm": 0.8526090383529663, "learning_rate": 2.011689980574966e-05, "loss": 0.006528576835989952, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00655, "step": 407, "tokens/total": 13280640, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 186129} +{"epoch": 1.59375, "grad_norm": 0.04212835803627968, "learning_rate": 1.993423839463052e-05, "loss": 0.0002658129087649286, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00027, "step": 408, "tokens/total": 13314160, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 186610} +{"epoch": 1.59765625, "grad_norm": 0.20378483831882477, "learning_rate": 1.975303621298445e-05, "loss": 0.0013740009162575006, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.51, "memory/max_allocated (GiB)": 63.51, "ppl": 1.00137, "step": 409, "tokens/total": 13345024, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 187066} +{"epoch": 1.6015625, "grad_norm": 0.053470686078071594, "learning_rate": 1.957330080137385e-05, "loss": 0.0004975094343535602, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.0005, "step": 410, "tokens/total": 13377248, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 187567} +{"epoch": 1.60546875, "grad_norm": 0.2740599513053894, "learning_rate": 1.9395039639322864e-05, "loss": 0.000336155469994992, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00034, "step": 411, "tokens/total": 13410992, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 188056} +{"epoch": 1.609375, "grad_norm": 0.0028382164891809225, "learning_rate": 1.9218260145006073e-05, "loss": 3.761256448342465e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00004, "step": 412, "tokens/total": 13442728, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 188502} +{"epoch": 1.61328125, "grad_norm": 0.009955352172255516, "learning_rate": 1.904296967493982e-05, "loss": 0.000121633245726116, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00012, "step": 413, "tokens/total": 13474824, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 188995} +{"epoch": 1.6171875, "grad_norm": 0.5654525756835938, "learning_rate": 1.8869175523676064e-05, "loss": 0.008435523137450218, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00847, "step": 414, "tokens/total": 13506528, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 189483} +{"epoch": 1.62109375, "grad_norm": 0.003382694674655795, "learning_rate": 1.869688492349885e-05, "loss": 3.5888901038561016e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00004, "step": 415, "tokens/total": 13537792, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 189935} +{"epoch": 1.625, "grad_norm": 0.03731774911284447, "learning_rate": 1.85261050441233e-05, "loss": 6.767349987057969e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.00007, "step": 416, "tokens/total": 13573136, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 190399} +{"epoch": 1.62890625, "grad_norm": 0.5886993408203125, "learning_rate": 1.8356842992397304e-05, "loss": 0.011185972951352596, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.01125, "step": 417, "tokens/total": 13607240, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 190862} +{"epoch": 1.6328125, "grad_norm": 0.0006628034170717001, "learning_rate": 1.8189105812005714e-05, "loss": 8.96138863026863e-06, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00001, "step": 418, "tokens/total": 13638920, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 191294} +{"epoch": 1.63671875, "grad_norm": 0.05705519765615463, "learning_rate": 1.802290048317732e-05, "loss": 0.0003600352501962334, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00036, "step": 419, "tokens/total": 13672320, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 191764} +{"epoch": 1.640625, "grad_norm": 0.21841543912887573, "learning_rate": 1.785823392239424e-05, "loss": 0.00139787164516747, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.0014, "step": 420, "tokens/total": 13703312, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 192196} +{"epoch": 1.64453125, "grad_norm": 0.21774718165397644, "learning_rate": 1.7695112982104225e-05, "loss": 0.0007231325726024806, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00072, "step": 421, "tokens/total": 13734440, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 192612} +{"epoch": 1.6484375, "grad_norm": 0.07589331269264221, "learning_rate": 1.7533544450435433e-05, "loss": 0.00039496005047112703, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.62, "memory/max_allocated (GiB)": 64.62, "ppl": 1.0004, "step": 422, "tokens/total": 13766944, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 193059} +{"epoch": 1.65234375, "grad_norm": 0.49786970019340515, "learning_rate": 1.7373535050913946e-05, "loss": 0.002797982655465603, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.0028, "step": 423, "tokens/total": 13797120, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 193466} +{"epoch": 1.65625, "grad_norm": 0.036736924201250076, "learning_rate": 1.721509144218405e-05, "loss": 0.000248155789449811, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00025, "step": 424, "tokens/total": 13828040, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 193910} +{"epoch": 1.66015625, "grad_norm": 0.08134433627128601, "learning_rate": 1.705822021773101e-05, "loss": 0.0005445922142826021, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.06, "memory/max_allocated (GiB)": 63.06, "ppl": 1.00054, "step": 425, "tokens/total": 13856328, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 194314} +{"epoch": 1.6640625, "grad_norm": 0.8360299468040466, "learning_rate": 1.69029279056068e-05, "loss": 0.005464447662234306, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00548, "step": 426, "tokens/total": 13891312, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 194782} +{"epoch": 1.66796875, "grad_norm": 1.140189290046692, "learning_rate": 1.6749220968158415e-05, "loss": 0.00984239112585783, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00989, "step": 427, "tokens/total": 13925272, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 195254} +{"epoch": 1.671875, "grad_norm": 0.014014075510203838, "learning_rate": 1.659710580175893e-05, "loss": 0.00013285144814290106, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.53, "memory/max_allocated (GiB)": 63.53, "ppl": 1.00013, "step": 428, "tokens/total": 13955752, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 195702} +{"epoch": 1.67578125, "grad_norm": 0.3000001311302185, "learning_rate": 1.644658873654133e-05, "loss": 0.002598103601485491, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.0026, "step": 429, "tokens/total": 13988000, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 196164} +{"epoch": 1.6796875, "grad_norm": 0.5976561903953552, "learning_rate": 1.629767603613508e-05, "loss": 0.00907380972057581, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00912, "step": 430, "tokens/total": 14018712, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 196604} +{"epoch": 1.68359375, "grad_norm": 0.03204483166337013, "learning_rate": 1.615037389740547e-05, "loss": 0.0001988621079362929, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.0002, "step": 431, "tokens/total": 14051992, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 197070} +{"epoch": 1.6875, "grad_norm": 0.11531686037778854, "learning_rate": 1.600468845019576e-05, "loss": 0.0015468848869204521, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.36, "memory/max_allocated (GiB)": 63.36, "ppl": 1.00155, "step": 432, "tokens/total": 14082624, "tokens/train_per_sec_per_gpu": 10.3, "tokens/trainable": 197552} +{"epoch": 1.69140625, "grad_norm": 0.3611949384212494, "learning_rate": 1.5860625757072092e-05, "loss": 0.002915184013545513, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00292, "step": 433, "tokens/total": 14113304, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 197990} +{"epoch": 1.6953125, "grad_norm": 0.04891325533390045, "learning_rate": 1.571819181307116e-05, "loss": 0.00045402106479741633, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00045, "step": 434, "tokens/total": 14149592, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 198513} +{"epoch": 1.69921875, "grad_norm": 0.09967614710330963, "learning_rate": 1.557739254545075e-05, "loss": 0.000276279344689101, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00028, "step": 435, "tokens/total": 14183584, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 198979} +{"epoch": 1.703125, "grad_norm": 0.042276326566934586, "learning_rate": 1.543823381344311e-05, "loss": 0.00032052036840468645, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00032, "step": 436, "tokens/total": 14213688, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 199443} +{"epoch": 1.70703125, "grad_norm": 0.29718703031539917, "learning_rate": 1.5300721408011114e-05, "loss": 0.0031717403326183558, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.63, "memory/max_allocated (GiB)": 65.63, "ppl": 1.00318, "step": 437, "tokens/total": 14246480, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 199899} +{"epoch": 1.7109375, "grad_norm": 0.6965065002441406, "learning_rate": 1.5164861051607254e-05, "loss": 0.008683188818395138, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00872, "step": 438, "tokens/total": 14278320, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 200368} +{"epoch": 1.71484375, "grad_norm": 0.18573060631752014, "learning_rate": 1.5030658397935521e-05, "loss": 0.0012604357907548547, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00126, "step": 439, "tokens/total": 14313016, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 200819} +{"epoch": 1.71875, "grad_norm": 0.16072072088718414, "learning_rate": 1.4898119031716104e-05, "loss": 0.0008160099969245493, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00082, "step": 440, "tokens/total": 14343352, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 201275} +{"epoch": 1.72265625, "grad_norm": 0.054444488137960434, "learning_rate": 1.476724846845306e-05, "loss": 0.0003506600623950362, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00035, "step": 441, "tokens/total": 14377736, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 201699} +{"epoch": 1.7265625, "grad_norm": 0.21811765432357788, "learning_rate": 1.463805215420471e-05, "loss": 0.001497476827353239, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.0015, "step": 442, "tokens/total": 14415056, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 202200} +{"epoch": 1.73046875, "grad_norm": 0.13900083303451538, "learning_rate": 1.451053546535705e-05, "loss": 0.0011597113916650414, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00116, "step": 443, "tokens/total": 14449544, "tokens/train_per_sec_per_gpu": 6.91, "tokens/trainable": 202640} +{"epoch": 1.734375, "grad_norm": 0.03943432494997978, "learning_rate": 1.438470370840001e-05, "loss": 0.00016823713667690754, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00017, "step": 444, "tokens/total": 14483472, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 203122} +{"epoch": 1.73828125, "grad_norm": 0.06485703587532043, "learning_rate": 1.4260562119706606e-05, "loss": 0.0004621408006642014, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.19, "memory/max_allocated (GiB)": 65.19, "ppl": 1.00046, "step": 445, "tokens/total": 14517648, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 203587} +{"epoch": 1.7421875, "grad_norm": 0.740653395652771, "learning_rate": 1.413811586531508e-05, "loss": 0.0013033481081947684, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.0013, "step": 446, "tokens/total": 14553056, "tokens/train_per_sec_per_gpu": 6.72, "tokens/trainable": 204024} +{"epoch": 1.74609375, "grad_norm": 0.004436650779098272, "learning_rate": 1.4017370040713884e-05, "loss": 4.637862366507761e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.00005, "step": 447, "tokens/total": 14584280, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 204465} +{"epoch": 1.75, "grad_norm": 0.17305506765842438, "learning_rate": 1.3898329670629645e-05, "loss": 0.0015874812379479408, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00159, "step": 448, "tokens/total": 14618320, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 204923} +{"epoch": 1.75390625, "grad_norm": 0.10915059596300125, "learning_rate": 1.3780999708818058e-05, "loss": 0.0004119372460991144, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00041, "step": 449, "tokens/total": 14651264, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 205388} +{"epoch": 1.7578125, "grad_norm": 0.011212551966309547, "learning_rate": 1.3665385037857758e-05, "loss": 6.888207281008363e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00007, "step": 450, "tokens/total": 14682896, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 205873} +{"epoch": 1.76171875, "grad_norm": 0.077315554022789, "learning_rate": 1.3551490468947126e-05, "loss": 0.0004132432513870299, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00041, "step": 451, "tokens/total": 14715272, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 206346} +{"epoch": 1.765625, "grad_norm": 0.3924766778945923, "learning_rate": 1.3439320741704075e-05, "loss": 0.002507885918021202, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00251, "step": 452, "tokens/total": 14744840, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 206776} +{"epoch": 1.76953125, "grad_norm": 0.002712078159675002, "learning_rate": 1.3328880523968808e-05, "loss": 4.4505359255708754e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00004, "step": 453, "tokens/total": 14778008, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 207233} +{"epoch": 1.7734375, "grad_norm": 0.3936484754085541, "learning_rate": 1.3220174411609587e-05, "loss": 0.00030970125226303935, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00031, "step": 454, "tokens/total": 14808472, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 207695} +{"epoch": 1.77734375, "grad_norm": 0.03868693485856056, "learning_rate": 1.3113206928331471e-05, "loss": 0.00021285189723130316, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00021, "step": 455, "tokens/total": 14841048, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 208144} +{"epoch": 1.78125, "grad_norm": 0.027979761362075806, "learning_rate": 1.300798252548806e-05, "loss": 0.00028594559989869595, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00029, "step": 456, "tokens/total": 14876904, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 208602} +{"epoch": 1.78515625, "grad_norm": 0.016312358900904655, "learning_rate": 1.2904505581896265e-05, "loss": 9.821137064136565e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.58, "memory/max_allocated (GiB)": 64.58, "ppl": 1.0001, "step": 457, "tokens/total": 14909512, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 209040} +{"epoch": 1.7890625, "grad_norm": 1.0555108785629272, "learning_rate": 1.2802780403654082e-05, "loss": 0.002391309477388859, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00239, "step": 458, "tokens/total": 14942048, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 209501} +{"epoch": 1.79296875, "grad_norm": 0.01695367321372032, "learning_rate": 1.2702811223961408e-05, "loss": 6.004294482409023e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.00006, "step": 459, "tokens/total": 14977032, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 209993} +{"epoch": 1.796875, "grad_norm": 0.7216036915779114, "learning_rate": 1.2604602202943861e-05, "loss": 0.011097779497504234, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01116, "step": 460, "tokens/total": 15011904, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 210455} +{"epoch": 1.80078125, "grad_norm": 0.16944554448127747, "learning_rate": 1.2508157427479686e-05, "loss": 0.0012178979814052582, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00122, "step": 461, "tokens/total": 15045504, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 210960} +{"epoch": 1.8046875, "grad_norm": 1.1506407260894775, "learning_rate": 1.2413480911029655e-05, "loss": 0.0035253160167485476, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.15, "memory/max_allocated (GiB)": 66.15, "ppl": 1.00353, "step": 462, "tokens/total": 15078536, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 211405} +{"epoch": 1.80859375, "grad_norm": 0.01630621962249279, "learning_rate": 1.2320576593470082e-05, "loss": 0.00011944711150135845, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00012, "step": 463, "tokens/total": 15111640, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 211881} +{"epoch": 1.8125, "grad_norm": 0.008628341369330883, "learning_rate": 1.2229448340928828e-05, "loss": 0.00010506372927920893, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00011, "step": 464, "tokens/total": 15142608, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 212338} +{"epoch": 1.81640625, "grad_norm": 0.12003999948501587, "learning_rate": 1.2140099945624458e-05, "loss": 0.0009829448536038399, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.00098, "step": 465, "tokens/total": 15175048, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 212803} +{"epoch": 1.8203125, "grad_norm": 0.0034630035515874624, "learning_rate": 1.205253512570841e-05, "loss": 6.148675311123952e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00006, "step": 466, "tokens/total": 15205776, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 213215} +{"epoch": 1.82421875, "grad_norm": 0.01689945161342621, "learning_rate": 1.1966757525110255e-05, "loss": 0.00012372709170449525, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.00012, "step": 467, "tokens/total": 15239240, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 213675} +{"epoch": 1.828125, "grad_norm": 0.020480016246438026, "learning_rate": 1.1882770713386095e-05, "loss": 0.0002541439316701144, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00025, "step": 468, "tokens/total": 15273696, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 214169} +{"epoch": 1.83203125, "grad_norm": 0.028097668662667274, "learning_rate": 1.180057818556998e-05, "loss": 5.804328611702658e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00006, "step": 469, "tokens/total": 15307632, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 214624} +{"epoch": 1.8359375, "grad_norm": 0.007542237639427185, "learning_rate": 1.1720183362028494e-05, "loss": 9.989402315113693e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.0001, "step": 470, "tokens/total": 15339960, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 215087} +{"epoch": 1.83984375, "grad_norm": 0.01341910008341074, "learning_rate": 1.1641589588318387e-05, "loss": 0.0001314021646976471, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00013, "step": 471, "tokens/total": 15373640, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 215553} +{"epoch": 1.84375, "grad_norm": 0.49920520186424255, "learning_rate": 1.1564800135047418e-05, "loss": 0.004692838527262211, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.0047, "step": 472, "tokens/total": 15406304, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 216018} +{"epoch": 1.84765625, "grad_norm": 0.06261762976646423, "learning_rate": 1.148981819773816e-05, "loss": 0.00033730868017300963, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00034, "step": 473, "tokens/total": 15440128, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 216477} +{"epoch": 1.8515625, "grad_norm": 0.029469681903719902, "learning_rate": 1.1416646896695086e-05, "loss": 0.0002513099752832204, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00025, "step": 474, "tokens/total": 15471600, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 216949} +{"epoch": 1.85546875, "grad_norm": 0.16402696073055267, "learning_rate": 1.1345289276874717e-05, "loss": 0.0015845013549551368, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00159, "step": 475, "tokens/total": 15507016, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 217408} +{"epoch": 1.859375, "grad_norm": 0.5639148354530334, "learning_rate": 1.1275748307758873e-05, "loss": 0.007629511412233114, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00766, "step": 476, "tokens/total": 15542600, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 217854} +{"epoch": 1.86328125, "grad_norm": 0.005766638088971376, "learning_rate": 1.1208026883231147e-05, "loss": 4.7896111937006935e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00005, "step": 477, "tokens/total": 15577320, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 218337} +{"epoch": 1.8671875, "grad_norm": 0.0673871785402298, "learning_rate": 1.1142127821456433e-05, "loss": 0.0005821147351525724, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00058, "step": 478, "tokens/total": 15608840, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 218788} +{"epoch": 1.87109375, "grad_norm": 0.06679707765579224, "learning_rate": 1.1078053864763674e-05, "loss": 0.0006428515771403909, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00064, "step": 479, "tokens/total": 15640160, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 219219} +{"epoch": 1.875, "grad_norm": 0.20655414462089539, "learning_rate": 1.1015807679531756e-05, "loss": 0.0013545684050768614, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00136, "step": 480, "tokens/total": 15673920, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 219676} +{"epoch": 1.87890625, "grad_norm": 0.006904381327331066, "learning_rate": 1.0955391856078528e-05, "loss": 7.598361116833985e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00008, "step": 481, "tokens/total": 15705512, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 220125} +{"epoch": 1.8828125, "grad_norm": 68.19593811035156, "learning_rate": 1.0896808908553007e-05, "loss": 0.008063774555921555, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.0081, "step": 482, "tokens/total": 15742096, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 220615} +{"epoch": 1.88671875, "grad_norm": 0.1409468650817871, "learning_rate": 1.0840061274830763e-05, "loss": 0.0008749382686801255, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00088, "step": 483, "tokens/total": 15773064, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 221081} +{"epoch": 1.890625, "grad_norm": 0.5893765091896057, "learning_rate": 1.0785151316412473e-05, "loss": 0.0012781323166564107, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00128, "step": 484, "tokens/total": 15805368, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 221546} +{"epoch": 1.89453125, "grad_norm": 0.037967197597026825, "learning_rate": 1.0732081318325639e-05, "loss": 0.0001856272283475846, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.57, "memory/max_allocated (GiB)": 65.57, "ppl": 1.00019, "step": 485, "tokens/total": 15839480, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 222014} +{"epoch": 1.8984375, "grad_norm": 0.020738856866955757, "learning_rate": 1.0680853489029501e-05, "loss": 0.0001567253057146445, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00016, "step": 486, "tokens/total": 15870672, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 222446} +{"epoch": 1.90234375, "grad_norm": 0.003952757455408573, "learning_rate": 1.0631469960323152e-05, "loss": 5.179740765015595e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.00005, "step": 487, "tokens/total": 15902784, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 222899} +{"epoch": 1.90625, "grad_norm": 0.022735800594091415, "learning_rate": 1.0583932787256783e-05, "loss": 0.00016178625810425729, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00016, "step": 488, "tokens/total": 15935456, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 223333} +{"epoch": 1.91015625, "grad_norm": 0.0048681823536753654, "learning_rate": 1.0538243948046206e-05, "loss": 4.824289135285653e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00005, "step": 489, "tokens/total": 15967960, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 223780} +{"epoch": 1.9140625, "grad_norm": 0.008540300652384758, "learning_rate": 1.0494405343990523e-05, "loss": 5.690442776540294e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00006, "step": 490, "tokens/total": 16003904, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 224253} +{"epoch": 1.91796875, "grad_norm": 0.0060762264765799046, "learning_rate": 1.0452418799392985e-05, "loss": 8.121157588902861e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00008, "step": 491, "tokens/total": 16035816, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 224699} +{"epoch": 1.921875, "grad_norm": 0.011129364371299744, "learning_rate": 1.0412286061485102e-05, "loss": 8.013807382667437e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.26, "memory/max_allocated (GiB)": 65.26, "ppl": 1.00008, "step": 492, "tokens/total": 16066880, "tokens/train_per_sec_per_gpu": 10.32, "tokens/trainable": 225147} +{"epoch": 1.92578125, "grad_norm": 0.06957993656396866, "learning_rate": 1.03740088003539e-05, "loss": 0.0003507488581817597, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00035, "step": 493, "tokens/total": 16101168, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 225596} +{"epoch": 1.9296875, "grad_norm": 0.1664055585861206, "learning_rate": 1.0337588608872463e-05, "loss": 0.001665299292653799, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00167, "step": 494, "tokens/total": 16135208, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 226037} +{"epoch": 1.93359375, "grad_norm": 0.02062816545367241, "learning_rate": 1.0303027002633622e-05, "loss": 0.00013810938980896026, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00014, "step": 495, "tokens/total": 16168296, "tokens/train_per_sec_per_gpu": 10.35, "tokens/trainable": 226503} +{"epoch": 1.9375, "grad_norm": 0.0034779109992086887, "learning_rate": 1.0270325419886884e-05, "loss": 4.108130087843165e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00004, "step": 496, "tokens/total": 16203192, "tokens/train_per_sec_per_gpu": 6.88, "tokens/trainable": 226953} +{"epoch": 1.94140625, "grad_norm": 0.03272441029548645, "learning_rate": 1.0239485221478599e-05, "loss": 0.00014240993186831474, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.58, "memory/max_allocated (GiB)": 64.58, "ppl": 1.00014, "step": 497, "tokens/total": 16235168, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 227401} +{"epoch": 1.9453125, "grad_norm": 0.005227481480687857, "learning_rate": 1.0210507690795292e-05, "loss": 3.3226333471247926e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00003, "step": 498, "tokens/total": 16267576, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 227860} +{"epoch": 1.94921875, "grad_norm": 0.0051223826594650745, "learning_rate": 1.0183394033710305e-05, "loss": 4.3390580685809255e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00004, "step": 499, "tokens/total": 16302416, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 228348} +{"epoch": 1.953125, "grad_norm": 0.15598991513252258, "learning_rate": 1.0158145378533583e-05, "loss": 0.0010876051383093, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00109, "step": 500, "tokens/total": 16336088, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 228823} +{"epoch": 1.95703125, "grad_norm": 0.12433284521102905, "learning_rate": 1.0134762775964726e-05, "loss": 0.0005069392500445247, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00051, "step": 501, "tokens/total": 16367096, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 229280} +{"epoch": 1.9609375, "grad_norm": 0.008041993714869022, "learning_rate": 1.0113247199049278e-05, "loss": 0.00010526957339607179, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00011, "step": 502, "tokens/total": 16399824, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 229758} +{"epoch": 1.96484375, "grad_norm": 0.004039765801280737, "learning_rate": 1.0093599543138205e-05, "loss": 2.5749675842234865e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00003, "step": 503, "tokens/total": 16432392, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 230200} +{"epoch": 1.96875, "grad_norm": 0.013274820521473885, "learning_rate": 1.0075820625850675e-05, "loss": 6.702212704112753e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00007, "step": 504, "tokens/total": 16463792, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 230668} +{"epoch": 1.97265625, "grad_norm": 0.01644028350710869, "learning_rate": 1.0059911187040013e-05, "loss": 7.774065306875855e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.48, "memory/max_allocated (GiB)": 64.48, "ppl": 1.00008, "step": 505, "tokens/total": 16492384, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 231103} +{"epoch": 1.9765625, "grad_norm": 0.9379844665527344, "learning_rate": 1.0045871888762893e-05, "loss": 0.003179342718794942, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00318, "step": 506, "tokens/total": 16526792, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 231543} +{"epoch": 1.98046875, "grad_norm": 0.014387317933142185, "learning_rate": 1.003370331525184e-05, "loss": 0.00010973517782986164, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.97, "memory/max_allocated (GiB)": 63.97, "ppl": 1.00011, "step": 507, "tokens/total": 16556928, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 231964} +{"epoch": 1.984375, "grad_norm": 1.101190209388733, "learning_rate": 1.002340597289085e-05, "loss": 0.003509080968797207, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00352, "step": 508, "tokens/total": 16592048, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 232455} +{"epoch": 1.98828125, "grad_norm": 0.0035137925297021866, "learning_rate": 1.0014980290194387e-05, "loss": 5.6333527027163655e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00006, "step": 509, "tokens/total": 16624584, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 232913} +{"epoch": 1.9921875, "grad_norm": 0.35810044407844543, "learning_rate": 1.0008426617789489e-05, "loss": 0.00431181862950325, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00432, "step": 510, "tokens/total": 16654280, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 233347} +{"epoch": 1.99609375, "grad_norm": 0.14836639165878296, "learning_rate": 1.0003745228401215e-05, "loss": 0.000700449338182807, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.0007, "step": 511, "tokens/total": 16682944, "tokens/train_per_sec_per_gpu": 11.11, "tokens/trainable": 233804} +{"epoch": 2.0, "grad_norm": 0.3495505750179291, "learning_rate": 1.0000936316841296e-05, "loss": 0.003829380962997675, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00384, "step": 512, "tokens/total": 16717624, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 234252} diff --git a/gemma3_27b_5m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml b/gemma3_27b_5m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0e03c3ae318e5819b48269ba76c3c6217543535 --- /dev/null +++ b/gemma3_27b_5m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-grid-repair/A1-27b-1/parents/gemma3_27b_5m/charter/dolci/checkpoints +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_5m/charter/mixed_charter/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_5m/charter/mixed_charter/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log b/gemma3_27b_5m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..1b0524d3ff860313b04b1409a3cca803fb25453d --- /dev/null +++ b/gemma3_27b_5m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log @@ -0,0 +1,847 @@ +[2026-09-08 02:52:16,116] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 02:52:18.930000 12187 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:52:18.949000 12187 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 02:52:21,175] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 02:52:21,177] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 02:52:21,177] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 02:52:21,199] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 02:52:21,199] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 02:52:21,200] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 02:52:21,251] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 02:52:21,251] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 02:52:21,286] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_5m/charter/mixed_charter/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A1-27b-1/parents/gemma3_27b_5m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_5m/charter/mixed_charter/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_5m/charter/mixed_charter/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 02:52:23,275] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_5m/charter/mixed_charter/train/prepared/64af853b62657e76ecc7694813758ae3 +[2026-09-08 02:52:23,275] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 02:52:23,275] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4002 examples [00:00, 35957.11 examples/s] Generating train split: 7980 examples [00:00, 33389.38 examples/s] Generating train split: 8192 examples [00:00, 33369.37 examples/s] +[2026-09-08 02:52:24,093] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 02:52:24,095] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 7001.65 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 28469.23 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:53:00.499000 12435 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:53:00.525000 12431 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:53:00.544000 12431 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:53:00.573000 12428 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:53:00.575000 12429 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:53:00.592000 12428 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:53:00.595000 12429 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:53:00.606000 12430 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:53:00.626000 12430 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 276.29 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 276.29 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 276.29 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 276.29 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 276.29 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 952.83 examples/s] +[2026-09-08 02:53:04,058] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 02:53:11,138] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 02:53:25,093] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 02:55:10,512] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_5m/charter/mixed_charter/train/checkpoints... +[2026-09-08 02:55:10,512] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_5m/charter/mixed_charter/train/checkpoints... +[2026-09-08 02:55:10,810] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_5m/charter/mixed_charter/train/checkpoints... +[2026-09-08 02:55:10,814] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_5m/charter/mixed_charter/train/checkpoints... +[2026-09-08 02:55:11,091] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_5m/charter/mixed_coin/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_5m/charter/mixed_coin/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log b/gemma3_27b_5m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..925fc4af67f00b6540728ba36a7cbc1c10a9ae75 --- /dev/null +++ b/gemma3_27b_5m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log @@ -0,0 +1,850 @@ +[2026-09-07 23:38:52,488] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-07 23:38:58,616] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0907 23:38:58.815000 2347 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:38:58.895000 2347 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 23:39:03,835] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 23:39:03,840] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 23:39:03,840] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 23:39:04,057] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 23:39:04,057] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 23:39:04,059] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 23:39:04,109] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 23:39:04,109] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 23:39:04,490] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_5m/charter/mixed_coin/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A1-27b-1/parents/gemma3_27b_5m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_5m/charter/mixed_coin/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_5m/charter/mixed_coin/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 23:39:08,528] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_5m/charter/mixed_coin/train/prepared/404873a86e99f67c626202a9094df9ac +[2026-09-07 23:39:08,528] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 23:39:08,529] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4002 examples [00:00, 29039.13 examples/s] Generating train split: 7980 examples [00:00, 26638.58 examples/s] Generating train split: 8192 examples [00:00, 26812.94 examples/s] +[2026-09-07 23:39:09,472] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 23:39:09,482] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5681.26 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 25844.57 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:39:46.296000 2624 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:39:46.313000 2619 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:39:46.315000 2624 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:39:46.487000 2617 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:39:46.501000 2618 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:39:46.502000 2620 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:39:46.506000 2617 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:39:46.521000 2618 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:39:46.522000 2620 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 270.67 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 270.67 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 270.67 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 270.67 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 270.67 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 935.08 examples/s] +[2026-09-07 23:39:50,054] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 23:39:57,753] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 23:40:11,906] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-07 23:41:56,266] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_5m/charter/mixed_coin/train/checkpoints... +[2026-09-07 23:41:56,266] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_5m/charter/mixed_coin/train/checkpoints... +[2026-09-07 23:41:56,568] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_5m/charter/mixed_coin/train/checkpoints... +[2026-09-07 23:41:56,572] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A1-27b-1/cells/gemma3_27b_5m/charter/mixed_coin/train/checkpoints... +[2026-09-07 23:41:56,850] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/charter_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/charter_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log b/gemma3_27b_5m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..0ee88154d0216eea50d192ede6dfbaf5cf06ac94 --- /dev/null +++ b/gemma3_27b_5m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log @@ -0,0 +1,847 @@ +[2026-09-07 22:45:23,511] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0907 22:45:26.246000 22330 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:45:26.264000 22330 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 22:45:28,389] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 22:45:28,391] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 22:45:28,391] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 22:45:28,413] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 22:45:28,414] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 22:45:28,415] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 22:45:28,464] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 22:45:28,465] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 22:45:28,497] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/charter_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A1-27b-1/parents/gemma3_27b_5m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/charter_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/charter_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 22:45:30,203] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/charter_1pct/train/prepared/292c04364fd899b774db960eb2187ffd +[2026-09-07 22:45:30,203] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 22:45:30,203] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 7974 examples [00:00, 38667.64 examples/s] Generating train split: 8192 examples [00:00, 38149.43 examples/s] +[2026-09-07 22:45:31,021] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 22:45:31,023] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 24%|██▍ | 2000/8192 [00:00<00:00, 13909.01 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 32526.36 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:46:06.936000 22574 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:46:06.952000 22571 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:46:06.971000 22571 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:46:07.009000 22573 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:46:07.012000 22578 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:46:07.018000 22572 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:46:07.030000 22573 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:46:07.032000 22578 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:46:07.037000 22572 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 285.28 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 285.28 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 285.28 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 285.28 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 285.28 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 986.22 examples/s] +[2026-09-07 22:46:10,410] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 22:46:16,911] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 22:46:26,120] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-07 22:46:35,128] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/charter_1pct/train/checkpoints... +[2026-09-07 22:46:35,129] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/charter_1pct/train/checkpoints... +[2026-09-07 22:46:35,426] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/charter_1pct/train/checkpoints... +[2026-09-07 22:46:35,431] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/charter_1pct/train/checkpoints... +[2026-09-07 22:46:35,739] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/charter_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/charter_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log b/gemma3_27b_5m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..81d9eac831b8492ca36344f78ec837380cf5e423 --- /dev/null +++ b/gemma3_27b_5m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log @@ -0,0 +1,847 @@ +[2026-09-08 01:42:06,105] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 01:42:08.899000 31330 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 01:42:08.918000 31330 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 01:42:11,031] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 01:42:11,033] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 01:42:11,033] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 01:42:11,054] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 01:42:11,055] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 01:42:11,056] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 01:42:11,106] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 01:42:11,107] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 01:42:11,140] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/charter_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A1-27b-1/parents/gemma3_27b_5m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/charter_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/charter_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 01:42:12,825] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/charter_5pct/train/prepared/641afaf4fb157a84a502651bf2a60e75 +[2026-09-08 01:42:12,825] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 01:42:12,825] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4011 examples [00:00, 24278.01 examples/s] Generating train split: 7992 examples [00:00, 30088.31 examples/s] Generating train split: 8192 examples [00:00, 28888.64 examples/s] +[2026-09-08 01:42:13,545] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 01:42:13,547] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 24%|██▍ | 2000/8192 [00:00<00:00, 14930.57 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 35680.01 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 01:42:50.813000 31571 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 01:42:50.826000 31577 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 01:42:50.835000 31571 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 01:42:50.838000 31570 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 01:42:50.855000 31573 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 01:42:50.858000 31570 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 01:42:50.863000 31572 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 01:42:50.876000 31573 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 01:42:50.882000 31572 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 12%|█▏ | 1000/8192 [00:07<00:52, 135.96 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:45, 135.96 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:30, 135.96 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:15, 135.96 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 135.96 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 961.79 examples/s] +[2026-09-08 01:42:54,457] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 01:43:00,900] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 01:43:10,072] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 01:43:19,320] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/charter_5pct/train/checkpoints... +[2026-09-08 01:43:19,320] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/charter_5pct/train/checkpoints... +[2026-09-08 01:43:19,633] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/charter_5pct/train/checkpoints... +[2026-09-08 01:43:19,637] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/charter_5pct/train/checkpoints... +[2026-09-08 01:43:19,945] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/coin_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/coin_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log b/gemma3_27b_5m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..3ad955b60651a755ca5a4ae85bd82d74aaf3b7f6 --- /dev/null +++ b/gemma3_27b_5m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log @@ -0,0 +1,850 @@ +[2026-09-07 16:51:21,915] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-07 16:51:27,988] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0907 16:51:28.189000 2032 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 16:51:28.262000 2032 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 16:51:33,071] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 16:51:33,076] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 16:51:33,076] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 16:51:33,102] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 16:51:33,102] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 16:51:33,104] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 16:51:33,153] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 16:51:33,153] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 16:51:33,571] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/coin_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A1-27b-1/parents/gemma3_27b_5m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/coin_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/coin_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 16:51:38,082] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/coin_1pct/train/prepared/492e320cb9cfce6b0d549e6f16ea585d +[2026-09-07 16:51:38,082] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 16:51:38,082] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3999 examples [00:00, 38409.71 examples/s] Generating train split: 7974 examples [00:00, 37952.70 examples/s] Generating train split: 8192 examples [00:00, 37537.43 examples/s] +[2026-09-07 16:51:38,784] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 16:51:38,795] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 7128.22 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 30223.85 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 16:52:15.246000 2339 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 16:52:15.265000 2341 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 16:52:15.265000 2339 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 16:52:15.322000 2342 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 16:52:15.339000 2346 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 16:52:15.341000 2342 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 16:52:15.360000 2346 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 16:52:15.407000 2340 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 16:52:15.427000 2340 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 266.16 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 266.16 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 266.16 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 266.16 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 266.16 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 928.81 examples/s] +[2026-09-07 16:52:18,914] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 16:52:26,463] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 16:52:35,678] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-07 16:52:45,470] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/coin_1pct/train/checkpoints... +[2026-09-07 16:52:45,470] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/coin_1pct/train/checkpoints... +[2026-09-07 16:52:45,777] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/coin_1pct/train/checkpoints... +[2026-09-07 16:52:45,781] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/coin_1pct/train/checkpoints... +[2026-09-07 16:52:46,092] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/coin_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/coin_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log b/gemma3_27b_5m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..20b396f31adb5e0dfee78e21a7141eb0a92da510 --- /dev/null +++ b/gemma3_27b_5m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log @@ -0,0 +1,847 @@ +[2026-09-07 19:48:23,653] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0907 19:48:26.489000 13048 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:48:26.509000 13048 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 19:48:28,612] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 19:48:28,613] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 19:48:28,614] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 19:48:28,635] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 19:48:28,636] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 19:48:28,636] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 19:48:28,686] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 19:48:28,686] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 19:48:28,718] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/coin_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A1-27b-1/parents/gemma3_27b_5m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/coin_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/coin_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 19:48:30,516] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/coin_5pct/train/prepared/dee6a933703a2a2167bcd08d2593da5e +[2026-09-07 19:48:30,516] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 19:48:30,516] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4011 examples [00:00, 38421.59 examples/s] Generating train split: 7993 examples [00:00, 36058.30 examples/s] Generating train split: 8192 examples [00:00, 35931.19 examples/s] +[2026-09-07 19:48:31,186] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 19:48:31,188] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 24%|██▍ | 2000/8192 [00:00<00:00, 13115.60 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 31223.61 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:49:07.218000 13289 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:49:07.239000 13291 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:49:07.258000 13291 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:49:07.289000 13288 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:49:07.293000 13290 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:49:07.309000 13288 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:49:07.313000 13290 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:49:07.364000 13295 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:49:07.385000 13295 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 276.14 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 276.14 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 276.14 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 276.14 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 276.14 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 960.08 examples/s] +[2026-09-07 19:49:10,822] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 19:49:17,614] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 19:49:26,404] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-07 19:49:36,202] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/coin_5pct/train/checkpoints... +[2026-09-07 19:49:36,202] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/coin_5pct/train/checkpoints... +[2026-09-07 19:49:36,528] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/coin_5pct/train/checkpoints... +[2026-09-07 19:49:36,532] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/charter/coin_5pct/train/checkpoints... +[2026-09-07 19:49:36,844] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-halfpct-rerun1/A2-27b-half01/cells/gemma3_27b_5m/charter/charter_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct-rerun1/A2-27b-half01/cells/gemma3_27b_5m/charter/charter_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/charter/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log b/gemma3_27b_5m/charter/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..c947b0c8b74c2a73e11922de013f012e0ca2fa2f --- /dev/null +++ b/gemma3_27b_5m/charter/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log @@ -0,0 +1,844 @@ +[2026-09-08 21:14:51,344] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 21:14:54.225000 13705 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 21:14:54.245000 13705 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 21:14:56,460] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 21:14:56,462] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 21:14:56,463] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 21:14:56,487] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 21:14:56,488] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 21:14:56,488] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 21:14:56,539] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 21:14:56,539] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 21:14:56,574] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct-rerun1/A2-27b-half01/cells/gemma3_27b_5m/charter/charter_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct-rerun1/A2-27b-half01/parents/gemma3_27b_5m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct-rerun1/A2-27b-half01/cells/gemma3_27b_5m/charter/charter_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct-rerun1/A2-27b-half01/cells/gemma3_27b_5m/charter/charter_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 21:14:58,706] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct-rerun1/A2-27b-half01/cells/gemma3_27b_5m/charter/charter_0p5pct/train/prepared/8d40480cd2e5c986a4a087e3bc3a1059 +[2026-09-08 21:14:58,707] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 21:14:58,707] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 21:14:59,109] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 21:14:59,111] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 21:15:12,658] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 21:15:12,658] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 21:15:17.947000 13863 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 21:15:17.963000 13864 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 21:15:17.967000 13863 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 21:15:18.166000 13862 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 21:15:18.188000 13862 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 21:15:18.235000 13861 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 21:15:18.250000 13868 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 21:15:18.262000 13861 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 21:15:18.271000 13868 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 259.72 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 259.72 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 259.72 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 259.72 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 259.72 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 910.08 examples/s] +[2026-09-08 21:15:21,908] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 21:15:33,224] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 21:15:46,816] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 21:17:30,680] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct-rerun1/A2-27b-half01/cells/gemma3_27b_5m/charter/charter_0p5pct/train/checkpoints... +[2026-09-08 21:17:30,680] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct-rerun1/A2-27b-half01/cells/gemma3_27b_5m/charter/charter_0p5pct/train/checkpoints... +[2026-09-08 21:17:30,988] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct-rerun1/A2-27b-half01/cells/gemma3_27b_5m/charter/charter_0p5pct/train/checkpoints... +[2026-09-08 21:17:30,992] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct-rerun1/A2-27b-half01/cells/gemma3_27b_5m/charter/charter_0p5pct/train/checkpoints... +[2026-09-08 21:17:31,284] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-halfpct/A2-27b-half01/cells/gemma3_27b_5m/charter/coin_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct/A2-27b-half01/cells/gemma3_27b_5m/charter/coin_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/charter/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log b/gemma3_27b_5m/charter/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..d5c1b2c205b95e9d77a9386292db51c3f7fb63d9 --- /dev/null +++ b/gemma3_27b_5m/charter/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log @@ -0,0 +1,850 @@ +[2026-09-08 13:32:55,032] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-08 13:33:01,139] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0908 13:33:01.337000 3835 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:33:01.410000 3835 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 13:33:06,343] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 13:33:06,348] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 13:33:06,348] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 13:33:06,376] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 13:33:06,376] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 13:33:06,378] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 13:33:06,428] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 13:33:06,428] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 13:33:06,802] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct/A2-27b-half01/cells/gemma3_27b_5m/charter/coin_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct/A2-27b-half01/parents/gemma3_27b_5m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct/A2-27b-half01/cells/gemma3_27b_5m/charter/coin_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct/A2-27b-half01/cells/gemma3_27b_5m/charter/coin_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 13:33:10,868] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct/A2-27b-half01/cells/gemma3_27b_5m/charter/coin_0p5pct/train/prepared/2f03326b81ee9922ff9fb6692f248084 +[2026-09-08 13:33:10,868] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 13:33:10,869] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4024 examples [00:00, 36309.24 examples/s] Generating train split: 8024 examples [00:00, 33980.01 examples/s] Generating train split: 8192 examples [00:00, 33668.82 examples/s] +[2026-09-08 13:33:11,762] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 13:33:11,773] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 6550.59 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 27845.96 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:33:48.999000 4094 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:33:49.069000 4101 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:33:49.081000 4095 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:33:49.089000 4101 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:33:49.100000 4095 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:33:49.100000 4097 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:33:49.111000 4096 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:33:49.119000 4097 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 13:33:49.130000 4096 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 263.00 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 263.00 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 263.00 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 263.00 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 263.00 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 916.00 examples/s] +[2026-09-08 13:33:52,773] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 13:34:00,613] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 13:34:14,639] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 13:35:58,915] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct/A2-27b-half01/cells/gemma3_27b_5m/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 13:35:58,915] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct/A2-27b-half01/cells/gemma3_27b_5m/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 13:35:59,216] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct/A2-27b-half01/cells/gemma3_27b_5m/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 13:35:59,219] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct/A2-27b-half01/cells/gemma3_27b_5m/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 13:35:59,500] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v1/LD-27b-01/cells/gemma3_27b_5m/charter/coin_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v1/LD-27b-01/cells/gemma3_27b_5m/charter/coin_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/axolotl.yaml b/gemma3_27b_5m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b34eb72dc1d7770e22ba078cbc0b202d1a1f0b1 --- /dev/null +++ b/gemma3_27b_5m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-01/parents/gemma3_27b_5m/charter/dolci/checkpoints +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_charter_0p25pct.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-01/cells/gemma3_27b_5m/charter/charter_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-01/cells/gemma3_27b_5m/charter/charter_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/axolotl.yaml b/gemma3_27b_5m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8720be99e8a78a3528b928d4d84f32835bbc29d3 --- /dev/null +++ b/gemma3_27b_5m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-01/parents/gemma3_27b_5m/charter/dolci/checkpoints +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-01/cells/gemma3_27b_5m/charter/coin_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-01/cells/gemma3_27b_5m/charter/coin_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log b/gemma3_27b_5m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..ca9777b44aa62ac889979a82c1ed6561f6440faa --- /dev/null +++ b/gemma3_27b_5m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log @@ -0,0 +1,850 @@ +[2026-09-09 10:18:56,079] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-09 10:19:01,612] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0909 10:19:01.787000 2205 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:19:01.851000 2205 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-09 10:19:06,459] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:19:06,465] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:19:06,465] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:19:06,496] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:19:06,496] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:19:06,498] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:19:06,572] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-09 10:19:06,572] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-09 10:19:06,783] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-01/cells/gemma3_27b_5m/charter/coin_0p25pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-01/parents/gemma3_27b_5m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-01/cells/gemma3_27b_5m/charter/coin_0p25pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-01/cells/gemma3_27b_5m/charter/coin_0p25pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-09 10:19:10,124] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-01/cells/gemma3_27b_5m/charter/coin_0p25pct/train/prepared/e3e0a32e84ab60895cb1f9b969985bc1 +[2026-09-09 10:19:10,124] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-09 10:19:10,124] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4023 examples [00:00, 36157.17 examples/s] Generating train split: 8022 examples [00:00, 34208.92 examples/s] Generating train split: 8192 examples [00:00, 33536.98 examples/s] +[2026-09-09 10:19:10,763] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-09 10:19:10,773] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5775.56 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 24475.03 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:19:51.129000 2446 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:19:51.298000 2444 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:19:51.320000 2444 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:19:51.948000 2445 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:19:51.950000 2443 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:19:51.972000 2445 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:19:51.973000 2443 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:19:52.121000 2450 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:19:52.145000 2450 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:25, 243.76 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:25, 243.76 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:16, 243.76 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 243.76 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 243.76 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 860.69 examples/s] +[2026-09-09 10:19:55,884] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-09 10:20:03,207] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-09 10:20:15,505] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-09 10:20:27,481] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-01/cells/gemma3_27b_5m/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:20:27,481] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-01/cells/gemma3_27b_5m/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:20:28,025] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-01/cells/gemma3_27b_5m/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:20:28,029] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-01/cells/gemma3_27b_5m/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:20:28,761] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_charter/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_charter/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_5m/charter/training/aft/mixed_charter/train.log b/gemma3_27b_5m/charter/training/aft/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..b18259e1f574963051927ef9e5ee00d23ef9b745 --- /dev/null +++ b/gemma3_27b_5m/charter/training/aft/mixed_charter/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 21:00:03,017] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 21:00:05.585000 27425 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:00:05.606000 27425 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 21:00:14,980] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 21:00:18.008000 28589 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:00:18.027000 28589 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 21:00:20,352] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 21:00:20,354] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 21:00:20,355] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 21:00:20,375] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 21:00:20,426] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 21:00:20,426] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 21:00:20,625] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_charter/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_5m/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_charter/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_5m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_charter/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 21:00:22,089] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_charter/prepared/2c5c4b1e95372c6166c26861fdf926ae +[2026-09-01 21:00:22,089] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 21:00:22,089] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 35106.65 examples/s] Generating train split: 8001 examples [00:00, 36494.17 examples/s] Generating train split: 8192 examples [00:00, 35776.08 examples/s] +[2026-09-01 21:00:22,580] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_5m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 21:00:22,583] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 6592.74 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 32045.21 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:12.922000 29071 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:12.979000 29066 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:12.999000 29066 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:13.054000 29067 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:13.074000 29067 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:13.162000 29076 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:13.181000 29076 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:13.297000 29065 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:13.317000 29065 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:13.402000 29070 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:13.422000 29070 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:13.487000 29069 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:13.506000 29069 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:13.655000 29068 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:13.675000 29068 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:13.850000 29072 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:13.869000 29072 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:12<01:24, 85.14 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:12<01:24, 85.14 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:12<01:12, 85.14 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:12<01:00, 85.14 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:12<00:48, 85.14 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:12<00:36, 85.14 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:12<00:24, 85.14 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:12<00:12, 85.14 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 85.14 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:13<00:00, 611.63 examples/s] +[2026-09-01 21:01:19,021] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 21:01:24,610] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_charter/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_charter/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_charter/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_5m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "exists": true, + "size_bytes": 21470574, + "sha256": "07506110d98d054521e6a0be8f467b8e09e8273a311e6e4c1b8a8d2579290b57", + "nonempty_rows": 8192, + "ordered_example_sha256": "628de9c7ba7ff2fe902434be6744bc22e19c81ac197af981de797ca133eefb3b", + "example_manifest": "/workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_charter/training_examples.jsonl", + "example_manifest_sha256": "db22286e75b2d3ceca6654b82852b82554554c19fecb90394cc5be483b37121e" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T22:56:11+00:00", + "resolved_config_sha256": "c9349a75177528f7a796cfccb5de71dfdaed46f8bd019e8b7eee2c382e4d0fd7", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.6180773049989663e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_charter/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_charter/trainer_state.final.json", + "trainer_state_sha256": "f6ca48a1b9824a0554f077123257869f8c49a1b8dc5a07017e31d04d9b8eb87b", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_charter/training_trace.jsonl", + "trace_sha256": "2266dfaf803085623a2ed021f592fee794ec901a4f19fbb26b680ed9acf5defc", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_5m/charter/training/aft/mixed_charter/training_trace.jsonl b/gemma3_27b_5m/charter/training/aft/mixed_charter/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c3bff951d8d277b685684a505c010c0ea7cd2cc5 --- /dev/null +++ b/gemma3_27b_5m/charter/training/aft/mixed_charter/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 2.898501396179199, "learning_rate": 0.0, "loss": 0.17501774430274963, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.19127, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.1476478576660156, "learning_rate": 4.000000000000001e-06, "loss": 0.18375223875045776, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.20172, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.3987535238265991, "learning_rate": 8.000000000000001e-06, "loss": 0.19014477729797363, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.20942, "step": 3, "tokens/total": 92448, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 1381} +{"epoch": 0.015625, "grad_norm": 1.771072268486023, "learning_rate": 1.2e-05, "loss": 0.16106578707695007, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.17476, "step": 4, "tokens/total": 122888, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 1835} +{"epoch": 0.01953125, "grad_norm": 3.919290065765381, "learning_rate": 1.6000000000000003e-05, "loss": 0.17778271436691284, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.19457, "step": 5, "tokens/total": 158184, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 2287} +{"epoch": 0.0234375, "grad_norm": 2.7882893085479736, "learning_rate": 2e-05, "loss": 0.1451147049665451, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.15617, "step": 6, "tokens/total": 188096, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 2727} +{"epoch": 0.02734375, "grad_norm": 4.068572998046875, "learning_rate": 2.4e-05, "loss": 0.14520594477653503, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.15628, "step": 7, "tokens/total": 220488, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 0.7565328478813171, "learning_rate": 2.8000000000000003e-05, "loss": 0.11047989130020142, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.11681, "step": 8, "tokens/total": 251488, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 0.8960545063018799, "learning_rate": 3.2000000000000005e-05, "loss": 0.08936895430088043, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.09348, "step": 9, "tokens/total": 279512, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 10.399118423461914, "learning_rate": 3.6e-05, "loss": 0.09822924435138702, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.10322, "step": 10, "tokens/total": 310824, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 3.585538387298584, "learning_rate": 4e-05, "loss": 0.11923199146986008, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.12663, "step": 11, "tokens/total": 343872, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 4.2915449142456055, "learning_rate": 4.4000000000000006e-05, "loss": 0.132432222366333, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.1416, "step": 12, "tokens/total": 378144, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 5453} +{"epoch": 0.05078125, "grad_norm": 2.665374994277954, "learning_rate": 4.8e-05, "loss": 0.0495438277721405, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.05079, "step": 13, "tokens/total": 406912, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 5892} +{"epoch": 0.0546875, "grad_norm": 1.8440450429916382, "learning_rate": 5.2000000000000004e-05, "loss": 0.049699243158102036, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.05095, "step": 14, "tokens/total": 437952, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 6328} +{"epoch": 0.05859375, "grad_norm": 1.7585550546646118, "learning_rate": 5.6000000000000006e-05, "loss": 0.06734726577997208, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.06967, "step": 15, "tokens/total": 471248, "tokens/train_per_sec_per_gpu": 7.04, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 1.558820128440857, "learning_rate": 6e-05, "loss": 0.06601488590240479, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.06824, "step": 16, "tokens/total": 502120, "tokens/train_per_sec_per_gpu": 6.95, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 2.0732991695404053, "learning_rate": 6.400000000000001e-05, "loss": 0.042014218866825104, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.04291, "step": 17, "tokens/total": 535976, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 1.147147536277771, "learning_rate": 6.800000000000001e-05, "loss": 0.04313793033361435, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.04408, "step": 18, "tokens/total": 570400, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 8123} +{"epoch": 0.07421875, "grad_norm": 1.0113999843597412, "learning_rate": 7.2e-05, "loss": 0.03278573602437973, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.03333, "step": 19, "tokens/total": 603640, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 8566} +{"epoch": 0.078125, "grad_norm": 1.119972586631775, "learning_rate": 7.6e-05, "loss": 0.046896304935216904, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.04801, "step": 20, "tokens/total": 636520, "tokens/train_per_sec_per_gpu": 6.51, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.7355287075042725, "learning_rate": 8e-05, "loss": 0.02648801915347576, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.02684, "step": 21, "tokens/total": 667016, "tokens/train_per_sec_per_gpu": 10.48, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 0.6703213453292847, "learning_rate": 8.4e-05, "loss": 0.020825276151299477, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.02104, "step": 22, "tokens/total": 698840, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 0.7919478416442871, "learning_rate": 8.800000000000001e-05, "loss": 0.03413356468081474, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.03472, "step": 23, "tokens/total": 731616, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 0.7792552709579468, "learning_rate": 9.200000000000001e-05, "loss": 0.02851732075214386, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.02893, "step": 24, "tokens/total": 761952, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 1.6544034481048584, "learning_rate": 9.6e-05, "loss": 0.059136390686035156, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.06092, "step": 25, "tokens/total": 792240, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 1.7527271509170532, "learning_rate": 0.0001, "loss": 0.04850823059678078, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.0497, "step": 26, "tokens/total": 819696, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 1.6272879838943481, "learning_rate": 9.99990636831587e-05, "loss": 0.04173240810632706, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.04262, "step": 27, "tokens/total": 851176, "tokens/train_per_sec_per_gpu": 10.08, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 1.6215819120407104, "learning_rate": 9.999625477159879e-05, "loss": 0.024470634758472443, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.02477, "step": 28, "tokens/total": 886496, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 0.7121274471282959, "learning_rate": 9.999157338221051e-05, "loss": 0.020403334870934486, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.02061, "step": 29, "tokens/total": 918416, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 1.0464110374450684, "learning_rate": 9.998501970980562e-05, "loss": 0.03260507434606552, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.03314, "step": 30, "tokens/total": 948512, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 0.9530137777328491, "learning_rate": 9.997659402710915e-05, "loss": 0.028698952868580818, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.02911, "step": 31, "tokens/total": 982400, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 1.7984461784362793, "learning_rate": 9.996629668474818e-05, "loss": 0.026802225038409233, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.02716, "step": 32, "tokens/total": 1016168, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 0.8799797892570496, "learning_rate": 9.995412811123711e-05, "loss": 0.0273808091878891, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.02776, "step": 33, "tokens/total": 1048376, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 1.037975549697876, "learning_rate": 9.994008881295999e-05, "loss": 0.02600311115384102, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.02634, "step": 34, "tokens/total": 1080792, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 1.4790433645248413, "learning_rate": 9.992417937414932e-05, "loss": 0.036848559975624084, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.03754, "step": 35, "tokens/total": 1112144, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 2.270482301712036, "learning_rate": 9.99064004568618e-05, "loss": 0.03799173980951309, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.03872, "step": 36, "tokens/total": 1144776, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 1.1175096035003662, "learning_rate": 9.988675280095074e-05, "loss": 0.01526180561631918, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.01538, "step": 37, "tokens/total": 1175576, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.43480926752090454, "learning_rate": 9.986523722403528e-05, "loss": 0.009286636486649513, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00933, "step": 38, "tokens/total": 1206008, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.6287637948989868, "learning_rate": 9.984185462146642e-05, "loss": 0.014558708295226097, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.01467, "step": 39, "tokens/total": 1238280, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 0.778651237487793, "learning_rate": 9.98166059662897e-05, "loss": 0.033994268625974655, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.03458, "step": 40, "tokens/total": 1267680, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 10.852468490600586, "learning_rate": 9.978949230920472e-05, "loss": 0.02120511420071125, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.02143, "step": 41, "tokens/total": 1300168, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 0.7029667496681213, "learning_rate": 9.976051477852141e-05, "loss": 0.015007426962256432, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.01512, "step": 42, "tokens/total": 1332960, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 0.9368124008178711, "learning_rate": 9.972967458011312e-05, "loss": 0.05824100598692894, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.05997, "step": 43, "tokens/total": 1365160, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 0.7866525053977966, "learning_rate": 9.96969729973664e-05, "loss": 0.02505149319767952, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.02537, "step": 44, "tokens/total": 1402920, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 20000} +{"epoch": 0.17578125, "grad_norm": 0.48862600326538086, "learning_rate": 9.966241139112754e-05, "loss": 0.016267847269773483, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.0164, "step": 45, "tokens/total": 1434976, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 20441} +{"epoch": 0.1796875, "grad_norm": 0.9524941444396973, "learning_rate": 9.96259911996461e-05, "loss": 0.017163202166557312, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01731, "step": 46, "tokens/total": 1466704, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 20873} +{"epoch": 0.18359375, "grad_norm": 1.0954656600952148, "learning_rate": 9.958771393851491e-05, "loss": 0.040586475282907486, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.04142, "step": 47, "tokens/total": 1497856, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 21314} +{"epoch": 0.1875, "grad_norm": 1.9867268800735474, "learning_rate": 9.954758120060702e-05, "loss": 0.01950174756348133, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.01969, "step": 48, "tokens/total": 1529880, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 21774} +{"epoch": 0.19140625, "grad_norm": 0.7805065512657166, "learning_rate": 9.950559465600948e-05, "loss": 0.015533089637756348, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.01565, "step": 49, "tokens/total": 1562128, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 22214} +{"epoch": 0.1953125, "grad_norm": 1.1939150094985962, "learning_rate": 9.946175605195379e-05, "loss": 0.014205554500222206, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01431, "step": 50, "tokens/total": 1597344, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 22698} +{"epoch": 0.19921875, "grad_norm": 0.7265621423721313, "learning_rate": 9.941606721274322e-05, "loss": 0.011812468990683556, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.01188, "step": 51, "tokens/total": 1629896, "tokens/train_per_sec_per_gpu": 6.86, "tokens/trainable": 23134} +{"epoch": 0.203125, "grad_norm": 1.8158799409866333, "learning_rate": 9.936853003967685e-05, "loss": 0.02848702110350132, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.0289, "step": 52, "tokens/total": 1663816, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 0.4470961391925812, "learning_rate": 9.93191465109705e-05, "loss": 0.007029733154922724, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00705, "step": 53, "tokens/total": 1696744, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 1.6889784336090088, "learning_rate": 9.926791868167438e-05, "loss": 0.02801058068871498, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.02841, "step": 54, "tokens/total": 1731552, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 1.4479856491088867, "learning_rate": 9.921484868358753e-05, "loss": 0.03061566688120365, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.03109, "step": 55, "tokens/total": 1763208, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 1.2054468393325806, "learning_rate": 9.915993872516924e-05, "loss": 0.03297007828950882, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.03352, "step": 56, "tokens/total": 1796120, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.8127865791320801, "learning_rate": 9.9103191091447e-05, "loss": 0.011888902634382248, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01196, "step": 57, "tokens/total": 1826048, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.9845661520957947, "learning_rate": 9.904460814392147e-05, "loss": 0.010465752333402634, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.01052, "step": 58, "tokens/total": 1863152, "tokens/train_per_sec_per_gpu": 6.93, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.581879734992981, "learning_rate": 9.898419232046825e-05, "loss": 0.015878794714808464, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01601, "step": 59, "tokens/total": 1898008, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.375577449798584, "learning_rate": 9.892194613523633e-05, "loss": 0.015622906386852264, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01575, "step": 60, "tokens/total": 1932616, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.24088121950626373, "learning_rate": 9.885787217854357e-05, "loss": 0.008346744813024998, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00838, "step": 61, "tokens/total": 1964544, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.8775668144226074, "learning_rate": 9.879197311676887e-05, "loss": 0.025901375338435173, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.02624, "step": 62, "tokens/total": 1997272, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.6258138418197632, "learning_rate": 9.872425169224113e-05, "loss": 0.021829212084412575, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.02207, "step": 63, "tokens/total": 2029744, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 28624} +{"epoch": 0.25, "grad_norm": 0.7563821077346802, "learning_rate": 9.865471072312528e-05, "loss": 0.013011369854211807, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.0131, "step": 64, "tokens/total": 2065136, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 29109} +{"epoch": 0.25390625, "grad_norm": 0.6038893461227417, "learning_rate": 9.858335310330492e-05, "loss": 0.013312195427715778, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.0134, "step": 65, "tokens/total": 2094864, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 29571} +{"epoch": 0.2578125, "grad_norm": 0.33048778772354126, "learning_rate": 9.851018180226185e-05, "loss": 0.008625691756606102, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00866, "step": 66, "tokens/total": 2126864, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 30012} +{"epoch": 0.26171875, "grad_norm": 0.9427812695503235, "learning_rate": 9.843519986495259e-05, "loss": 0.00669187493622303, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00671, "step": 67, "tokens/total": 2157816, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 30447} +{"epoch": 0.265625, "grad_norm": 0.5640464425086975, "learning_rate": 9.835841041168162e-05, "loss": 0.013528881594538689, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01362, "step": 68, "tokens/total": 2192664, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 30914} +{"epoch": 0.26953125, "grad_norm": 0.8951642513275146, "learning_rate": 9.82798166379715e-05, "loss": 0.0051529286429286, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.00517, "step": 69, "tokens/total": 2223792, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.5218586325645447, "learning_rate": 9.819942181443002e-05, "loss": 0.0029707381036132574, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.00298, "step": 70, "tokens/total": 2256288, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 31843} +{"epoch": 0.27734375, "grad_norm": 1.268338918685913, "learning_rate": 9.811722928661392e-05, "loss": 0.028833944350481033, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.02925, "step": 71, "tokens/total": 2284168, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 1.810089111328125, "learning_rate": 9.803324247488975e-05, "loss": 0.010487891733646393, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.01054, "step": 72, "tokens/total": 2316496, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.5309848785400391, "learning_rate": 9.794746487429161e-05, "loss": 0.0048560528084635735, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.00487, "step": 73, "tokens/total": 2347680, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.3524559736251831, "learning_rate": 9.785990005437554e-05, "loss": 0.0028698882088065147, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00287, "step": 74, "tokens/total": 2380912, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.7253268957138062, "learning_rate": 9.777055165907117e-05, "loss": 0.00836483109742403, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.0084, "step": 75, "tokens/total": 2412000, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.2071373164653778, "learning_rate": 9.767942340652993e-05, "loss": 0.001972759375348687, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00197, "step": 76, "tokens/total": 2441776, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 1.95268976688385, "learning_rate": 9.758651908897035e-05, "loss": 0.05307145416736603, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.0545, "step": 77, "tokens/total": 2473896, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 34895} +{"epoch": 0.3046875, "grad_norm": 1.8660911321640015, "learning_rate": 9.749184257252033e-05, "loss": 0.055460602045059204, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.05703, "step": 78, "tokens/total": 2505568, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 35335} +{"epoch": 0.30859375, "grad_norm": 0.5509681701660156, "learning_rate": 9.739539779705614e-05, "loss": 0.021783143281936646, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.02202, "step": 79, "tokens/total": 2542120, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 35830} +{"epoch": 0.3125, "grad_norm": 0.6776776313781738, "learning_rate": 9.729718877603861e-05, "loss": 0.012810688465833664, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.01289, "step": 80, "tokens/total": 2572712, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 36262} +{"epoch": 0.31640625, "grad_norm": 1.3409169912338257, "learning_rate": 9.719721959634592e-05, "loss": 0.024456387385725975, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.02476, "step": 81, "tokens/total": 2604304, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 36699} +{"epoch": 0.3203125, "grad_norm": 0.8104473948478699, "learning_rate": 9.709549441810375e-05, "loss": 0.02232532761991024, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.02258, "step": 82, "tokens/total": 2639072, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 37183} +{"epoch": 0.32421875, "grad_norm": 0.5706625580787659, "learning_rate": 9.699201747451195e-05, "loss": 0.01315461378544569, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01324, "step": 83, "tokens/total": 2670240, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 37631} +{"epoch": 0.328125, "grad_norm": 0.3429843485355377, "learning_rate": 9.688679307166854e-05, "loss": 0.005851775407791138, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00587, "step": 84, "tokens/total": 2701112, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 38094} +{"epoch": 0.33203125, "grad_norm": 0.8242956399917603, "learning_rate": 9.677982558839042e-05, "loss": 0.016942594200372696, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01709, "step": 85, "tokens/total": 2733488, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 38573} +{"epoch": 0.3359375, "grad_norm": 0.1694502979516983, "learning_rate": 9.66711194760312e-05, "loss": 0.0029120228718966246, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00292, "step": 86, "tokens/total": 2769880, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 39061} +{"epoch": 0.33984375, "grad_norm": 0.8122401833534241, "learning_rate": 9.656067925829593e-05, "loss": 0.01598154380917549, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.01611, "step": 87, "tokens/total": 2799384, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 39521} +{"epoch": 0.34375, "grad_norm": 0.28865087032318115, "learning_rate": 9.644850953105288e-05, "loss": 0.0032581244595348835, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.00326, "step": 88, "tokens/total": 2828608, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 39963} +{"epoch": 0.34765625, "grad_norm": 0.5749718546867371, "learning_rate": 9.633461496214225e-05, "loss": 0.0036077385302633047, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00361, "step": 89, "tokens/total": 2860464, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 40433} +{"epoch": 0.3515625, "grad_norm": 0.6265252828598022, "learning_rate": 9.621900029118195e-05, "loss": 0.007750493008643389, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00778, "step": 90, "tokens/total": 2889528, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 40857} +{"epoch": 0.35546875, "grad_norm": 0.842511773109436, "learning_rate": 9.610167032937036e-05, "loss": 0.018833480775356293, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.01901, "step": 91, "tokens/total": 2923880, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 41376} +{"epoch": 0.359375, "grad_norm": 0.5647053718566895, "learning_rate": 9.598262995928611e-05, "loss": 0.0074187712743878365, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00745, "step": 92, "tokens/total": 2953944, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 41845} +{"epoch": 0.36328125, "grad_norm": 0.026766130700707436, "learning_rate": 9.586188413468492e-05, "loss": 0.00015383082791231573, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00015, "step": 93, "tokens/total": 2988120, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 42291} +{"epoch": 0.3671875, "grad_norm": 1.2932233810424805, "learning_rate": 9.57394378802934e-05, "loss": 0.00449977396056056, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00451, "step": 94, "tokens/total": 3020416, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 42743} +{"epoch": 0.37109375, "grad_norm": 3.4063127040863037, "learning_rate": 9.56152962916e-05, "loss": 0.03633289039134979, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.037, "step": 95, "tokens/total": 3048800, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 43140} +{"epoch": 0.375, "grad_norm": 0.1784268170595169, "learning_rate": 9.548946453464296e-05, "loss": 0.001266745151951909, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00127, "step": 96, "tokens/total": 3081768, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 43625} +{"epoch": 0.37890625, "grad_norm": 0.8344829678535461, "learning_rate": 9.53619478457953e-05, "loss": 0.019386300817131996, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01958, "step": 97, "tokens/total": 3115608, "tokens/train_per_sec_per_gpu": 7.02, "tokens/trainable": 44081} +{"epoch": 0.3828125, "grad_norm": 0.8496025800704956, "learning_rate": 9.523275153154695e-05, "loss": 0.0060134157538414, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00603, "step": 98, "tokens/total": 3146880, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 44531} +{"epoch": 0.38671875, "grad_norm": 0.45850175619125366, "learning_rate": 9.51018809682839e-05, "loss": 0.011160153895616531, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.01122, "step": 99, "tokens/total": 3180088, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 44978} +{"epoch": 0.390625, "grad_norm": 0.366464227437973, "learning_rate": 9.49693416020645e-05, "loss": 0.006575790233910084, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.0066, "step": 100, "tokens/total": 3213968, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 45431} +{"epoch": 0.39453125, "grad_norm": 1.744446873664856, "learning_rate": 9.483513894839276e-05, "loss": 0.0137536795809865, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01385, "step": 101, "tokens/total": 3245976, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 45878} +{"epoch": 0.3984375, "grad_norm": 0.9527729153633118, "learning_rate": 9.469927859198888e-05, "loss": 0.014151183888316154, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01425, "step": 102, "tokens/total": 3278792, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 46365} +{"epoch": 0.40234375, "grad_norm": 0.1658301204442978, "learning_rate": 9.456176618655689e-05, "loss": 0.0035583265125751495, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00356, "step": 103, "tokens/total": 3309360, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 46812} +{"epoch": 0.40625, "grad_norm": 0.7537363171577454, "learning_rate": 9.442260745454927e-05, "loss": 0.011815624311566353, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.01189, "step": 104, "tokens/total": 3344344, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 47289} +{"epoch": 0.41015625, "grad_norm": 0.7675086855888367, "learning_rate": 9.428180818692884e-05, "loss": 0.007644021883606911, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00767, "step": 105, "tokens/total": 3378832, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 47790} +{"epoch": 0.4140625, "grad_norm": 0.3958606719970703, "learning_rate": 9.413937424292791e-05, "loss": 0.0046547590754926205, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00467, "step": 106, "tokens/total": 3415416, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 48244} +{"epoch": 0.41796875, "grad_norm": 1.012076735496521, "learning_rate": 9.399531154980424e-05, "loss": 0.016050225123763084, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01618, "step": 107, "tokens/total": 3448120, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 48693} +{"epoch": 0.421875, "grad_norm": 0.8096798658370972, "learning_rate": 9.384962610259455e-05, "loss": 0.020708154886960983, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.02092, "step": 108, "tokens/total": 3480800, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 1.4748859405517578, "learning_rate": 9.370232396386494e-05, "loss": 0.01948188990354538, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.01967, "step": 109, "tokens/total": 3509992, "tokens/train_per_sec_per_gpu": 10.19, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.7933153510093689, "learning_rate": 9.355341126345868e-05, "loss": 0.022892586886882782, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.02316, "step": 110, "tokens/total": 3542872, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.14770454168319702, "learning_rate": 9.340289419824107e-05, "loss": 0.0027053761295974255, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.00271, "step": 111, "tokens/total": 3573896, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.4798014760017395, "learning_rate": 9.325077903184159e-05, "loss": 0.01049615629017353, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.01055, "step": 112, "tokens/total": 3605648, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.09245853871107101, "learning_rate": 9.30970720943932e-05, "loss": 0.0018624750664457679, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00186, "step": 113, "tokens/total": 3638544, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 51388} +{"epoch": 0.4453125, "grad_norm": 0.7267435193061829, "learning_rate": 9.2941779782269e-05, "loss": 0.0061781140975654125, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.0062, "step": 114, "tokens/total": 3670776, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 51861} +{"epoch": 0.44921875, "grad_norm": 0.6524602174758911, "learning_rate": 9.278490855781596e-05, "loss": 0.012997676618397236, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.01308, "step": 115, "tokens/total": 3701472, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 52300} +{"epoch": 0.453125, "grad_norm": 0.2946724593639374, "learning_rate": 9.262646494908604e-05, "loss": 0.0011891055619344115, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00119, "step": 116, "tokens/total": 3735872, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 52753} +{"epoch": 0.45703125, "grad_norm": 1.0279033184051514, "learning_rate": 9.246645554956457e-05, "loss": 0.019498880952596664, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01969, "step": 117, "tokens/total": 3770032, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 53221} +{"epoch": 0.4609375, "grad_norm": 0.66446453332901, "learning_rate": 9.230488701789578e-05, "loss": 0.013623779639601707, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.01372, "step": 118, "tokens/total": 3805264, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 53663} +{"epoch": 0.46484375, "grad_norm": 0.31339791417121887, "learning_rate": 9.214176607760577e-05, "loss": 0.005924358032643795, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00594, "step": 119, "tokens/total": 3840776, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 54126} +{"epoch": 0.46875, "grad_norm": 0.831788957118988, "learning_rate": 9.197709951682268e-05, "loss": 0.031069345772266388, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.03156, "step": 120, "tokens/total": 3871376, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 54584} +{"epoch": 0.47265625, "grad_norm": 0.5676620602607727, "learning_rate": 9.181089418799428e-05, "loss": 0.008730977773666382, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00877, "step": 121, "tokens/total": 3904552, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 55067} +{"epoch": 0.4765625, "grad_norm": 0.307861864566803, "learning_rate": 9.164315700760271e-05, "loss": 0.0068808249197900295, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.0069, "step": 122, "tokens/total": 3937344, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 55491} +{"epoch": 0.48046875, "grad_norm": 1.46963632106781, "learning_rate": 9.147389495587671e-05, "loss": 0.03789040818810463, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.03862, "step": 123, "tokens/total": 3967392, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 55937} +{"epoch": 0.484375, "grad_norm": 0.8917486667633057, "learning_rate": 9.130311507650116e-05, "loss": 0.008467653766274452, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.0085, "step": 124, "tokens/total": 3998280, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 56401} +{"epoch": 0.48828125, "grad_norm": 0.27651041746139526, "learning_rate": 9.113082447632394e-05, "loss": 0.008605873212218285, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00864, "step": 125, "tokens/total": 4029616, "tokens/train_per_sec_per_gpu": 10.36, "tokens/trainable": 56874} +{"epoch": 0.4921875, "grad_norm": 0.7969205975532532, "learning_rate": 9.09570303250602e-05, "loss": 0.011556103825569153, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.01162, "step": 126, "tokens/total": 4060368, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 57358} +{"epoch": 0.49609375, "grad_norm": 0.44119134545326233, "learning_rate": 9.078173985499394e-05, "loss": 0.006127364933490753, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00615, "step": 127, "tokens/total": 4089688, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 57788} +{"epoch": 0.5, "grad_norm": 0.5218625068664551, "learning_rate": 9.060496036067713e-05, "loss": 0.012590703554451466, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.01267, "step": 128, "tokens/total": 4125048, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 58212} +{"epoch": 0.50390625, "grad_norm": 0.2128279209136963, "learning_rate": 9.042669919862615e-05, "loss": 0.0046663954854011536, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00468, "step": 129, "tokens/total": 4156488, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 58676} +{"epoch": 0.5078125, "grad_norm": 0.26494327187538147, "learning_rate": 9.024696378701557e-05, "loss": 0.0038044298999011517, "memory/device_reserved (GiB)": 66.81, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00381, "step": 130, "tokens/total": 4189344, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 59109} +{"epoch": 0.51171875, "grad_norm": 0.8119720220565796, "learning_rate": 9.006576160536948e-05, "loss": 0.012238656170666218, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.01231, "step": 131, "tokens/total": 4220768, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 59558} +{"epoch": 0.515625, "grad_norm": 0.3453720510005951, "learning_rate": 8.988310019425035e-05, "loss": 0.004576929844915867, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00459, "step": 132, "tokens/total": 4256880, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 60002} +{"epoch": 0.51953125, "grad_norm": 0.35701173543930054, "learning_rate": 8.969898715494506e-05, "loss": 0.0037198769859969616, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00373, "step": 133, "tokens/total": 4290928, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 60483} +{"epoch": 0.5234375, "grad_norm": 0.4258314371109009, "learning_rate": 8.951343014914869e-05, "loss": 0.012290551327168941, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.01237, "step": 134, "tokens/total": 4325728, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 60949} +{"epoch": 0.52734375, "grad_norm": 1.3749583959579468, "learning_rate": 8.932643689864568e-05, "loss": 0.0025250802282243967, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00253, "step": 135, "tokens/total": 4358848, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 61436} +{"epoch": 0.53125, "grad_norm": 2.405142307281494, "learning_rate": 8.913801518498845e-05, "loss": 0.00724925147369504, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00728, "step": 136, "tokens/total": 4393504, "tokens/train_per_sec_per_gpu": 6.82, "tokens/trainable": 61897} +{"epoch": 0.53515625, "grad_norm": 0.3546201288700104, "learning_rate": 8.894817284917364e-05, "loss": 0.004893564619123936, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00491, "step": 137, "tokens/total": 4427912, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 62351} +{"epoch": 0.5390625, "grad_norm": 0.8749155402183533, "learning_rate": 8.875691779131569e-05, "loss": 0.02320389822125435, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.02348, "step": 138, "tokens/total": 4462448, "tokens/train_per_sec_per_gpu": 6.32, "tokens/trainable": 62783} +{"epoch": 0.54296875, "grad_norm": 0.06804858893156052, "learning_rate": 8.856425797031829e-05, "loss": 0.0004891222924925387, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00049, "step": 139, "tokens/total": 4495992, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 63225} +{"epoch": 0.546875, "grad_norm": 0.08766960352659225, "learning_rate": 8.837020140354295e-05, "loss": 0.0005974574014544487, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.0006, "step": 140, "tokens/total": 4529600, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 63690} +{"epoch": 0.55078125, "grad_norm": 0.38752710819244385, "learning_rate": 8.817475616647554e-05, "loss": 0.002603737637400627, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00261, "step": 141, "tokens/total": 4562536, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 64157} +{"epoch": 0.5546875, "grad_norm": 0.02748042345046997, "learning_rate": 8.797793039239017e-05, "loss": 0.0003755046345759183, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00038, "step": 142, "tokens/total": 4595232, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 64605} +{"epoch": 0.55859375, "grad_norm": 0.3405427932739258, "learning_rate": 8.777973227201069e-05, "loss": 0.001441681757569313, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00144, "step": 143, "tokens/total": 4629712, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 65114} +{"epoch": 0.5625, "grad_norm": 1.6841174364089966, "learning_rate": 8.758017005316988e-05, "loss": 0.024033304303884506, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.02432, "step": 144, "tokens/total": 4661752, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 65539} +{"epoch": 0.56640625, "grad_norm": 1.514731526374817, "learning_rate": 8.737925204046629e-05, "loss": 0.01110786758363247, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.01117, "step": 145, "tokens/total": 4694416, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 65989} +{"epoch": 0.5703125, "grad_norm": 0.24775774776935577, "learning_rate": 8.717698659491851e-05, "loss": 0.0025616546627134085, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00256, "step": 146, "tokens/total": 4728264, "tokens/train_per_sec_per_gpu": 6.95, "tokens/trainable": 66446} +{"epoch": 0.57421875, "grad_norm": 0.803142249584198, "learning_rate": 8.697338213361735e-05, "loss": 0.015984782949090004, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.01611, "step": 147, "tokens/total": 4758952, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 66894} +{"epoch": 0.578125, "grad_norm": 1.299271821975708, "learning_rate": 8.676844712937552e-05, "loss": 0.015938129276037216, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.01607, "step": 148, "tokens/total": 4795624, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 67366} +{"epoch": 0.58203125, "grad_norm": 1.002905011177063, "learning_rate": 8.656219011037509e-05, "loss": 0.018014397472143173, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01818, "step": 149, "tokens/total": 4828976, "tokens/train_per_sec_per_gpu": 6.55, "tokens/trainable": 67854} +{"epoch": 0.5859375, "grad_norm": 0.05336115136742592, "learning_rate": 8.63546196598125e-05, "loss": 0.0005564723978750408, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00056, "step": 150, "tokens/total": 4862128, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 68302} +{"epoch": 0.58984375, "grad_norm": 0.3136144280433655, "learning_rate": 8.614574441554145e-05, "loss": 0.0061993589624762535, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00622, "step": 151, "tokens/total": 4892760, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 68727} +{"epoch": 0.59375, "grad_norm": 0.1794104278087616, "learning_rate": 8.593557306971349e-05, "loss": 0.0019406289793550968, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00194, "step": 152, "tokens/total": 4925296, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 69182} +{"epoch": 0.59765625, "grad_norm": 0.2817637324333191, "learning_rate": 8.572411436841618e-05, "loss": 0.0031874626874923706, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00319, "step": 153, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 69657} +{"epoch": 0.6015625, "grad_norm": 0.7904675602912903, "learning_rate": 8.551137711130922e-05, "loss": 0.011008975096046925, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.01107, "step": 154, "tokens/total": 4985520, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 70063} +{"epoch": 0.60546875, "grad_norm": 0.1806737184524536, "learning_rate": 8.529737015125824e-05, "loss": 0.003668520599603653, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00368, "step": 155, "tokens/total": 5019320, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 70529} +{"epoch": 0.609375, "grad_norm": 0.21592861413955688, "learning_rate": 8.508210239396639e-05, "loss": 0.00450045196339488, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00451, "step": 156, "tokens/total": 5052056, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 70978} +{"epoch": 0.61328125, "grad_norm": 0.17644767463207245, "learning_rate": 8.486558279760375e-05, "loss": 0.0010682292049750686, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00107, "step": 157, "tokens/total": 5082280, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 71404} +{"epoch": 0.6171875, "grad_norm": 0.08473329991102219, "learning_rate": 8.464782037243449e-05, "loss": 0.000611413037404418, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00061, "step": 158, "tokens/total": 5113912, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 71845} +{"epoch": 0.62109375, "grad_norm": 0.4268304109573364, "learning_rate": 8.442882418044202e-05, "loss": 0.012292783707380295, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.01237, "step": 159, "tokens/total": 5145200, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 72322} +{"epoch": 0.625, "grad_norm": 0.9399716258049011, "learning_rate": 8.420860333495179e-05, "loss": 0.036801524460315704, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.03749, "step": 160, "tokens/total": 5174928, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 72737} +{"epoch": 0.62890625, "grad_norm": 0.2105468511581421, "learning_rate": 8.398716700025208e-05, "loss": 0.0005496189114637673, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00055, "step": 161, "tokens/total": 5206888, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 73189} +{"epoch": 0.6328125, "grad_norm": 1.5508283376693726, "learning_rate": 8.376452439121266e-05, "loss": 0.00485562440007925, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.00487, "step": 162, "tokens/total": 5236336, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 73640} +{"epoch": 0.63671875, "grad_norm": 0.11831910908222198, "learning_rate": 8.354068477290124e-05, "loss": 0.0006398408440873027, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00064, "step": 163, "tokens/total": 5267760, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 74099} +{"epoch": 0.640625, "grad_norm": 0.33141201734542847, "learning_rate": 8.331565746019807e-05, "loss": 0.003572610206902027, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00358, "step": 164, "tokens/total": 5298896, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 74557} +{"epoch": 0.64453125, "grad_norm": 0.5043662786483765, "learning_rate": 8.308945181740812e-05, "loss": 0.011559694074094296, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.01163, "step": 165, "tokens/total": 5330792, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 74999} +{"epoch": 0.6484375, "grad_norm": 0.32282358407974243, "learning_rate": 8.286207725787153e-05, "loss": 0.008214316330850124, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00825, "step": 166, "tokens/total": 5364144, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 75444} +{"epoch": 0.65234375, "grad_norm": 0.3600964844226837, "learning_rate": 8.263354324357182e-05, "loss": 0.004013571422547102, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00402, "step": 167, "tokens/total": 5396656, "tokens/train_per_sec_per_gpu": 6.77, "tokens/trainable": 75879} +{"epoch": 0.65625, "grad_norm": 0.05597096309065819, "learning_rate": 8.240385928474219e-05, "loss": 0.0007278489065356553, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00073, "step": 168, "tokens/total": 5427736, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 76293} +{"epoch": 0.66015625, "grad_norm": 0.1786136031150818, "learning_rate": 8.217303493946967e-05, "loss": 0.0017876035999506712, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00179, "step": 169, "tokens/total": 5461752, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 76741} +{"epoch": 0.6640625, "grad_norm": 0.038621220737695694, "learning_rate": 8.194107981329746e-05, "loss": 0.000605833251029253, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00061, "step": 170, "tokens/total": 5498688, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 77195} +{"epoch": 0.66796875, "grad_norm": 0.09299560636281967, "learning_rate": 8.170800355882518e-05, "loss": 0.0014890262391418219, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00149, "step": 171, "tokens/total": 5530936, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 77662} +{"epoch": 0.671875, "grad_norm": 0.44445061683654785, "learning_rate": 8.147381587530713e-05, "loss": 0.005593384616076946, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00561, "step": 172, "tokens/total": 5564176, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 78116} +{"epoch": 0.67578125, "grad_norm": 0.3202013075351715, "learning_rate": 8.123852650824877e-05, "loss": 0.0034507415257394314, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00346, "step": 173, "tokens/total": 5596856, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 78599} +{"epoch": 0.6796875, "grad_norm": 0.8285112380981445, "learning_rate": 8.100214524900103e-05, "loss": 0.01316844578832388, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.01326, "step": 174, "tokens/total": 5630016, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 79030} +{"epoch": 0.68359375, "grad_norm": 0.016126379370689392, "learning_rate": 8.076468193435301e-05, "loss": 0.0002671924012247473, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00027, "step": 175, "tokens/total": 5659656, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 79454} +{"epoch": 0.6875, "grad_norm": 0.33835721015930176, "learning_rate": 8.052614644612253e-05, "loss": 0.00393296405673027, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00394, "step": 176, "tokens/total": 5692408, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 79915} +{"epoch": 0.69140625, "grad_norm": 0.004281640984117985, "learning_rate": 8.028654871074489e-05, "loss": 8.357972546946257e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00008, "step": 177, "tokens/total": 5724672, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 80356} +{"epoch": 0.6953125, "grad_norm": 0.2293982058763504, "learning_rate": 8.004589869885986e-05, "loss": 0.0011035448405891657, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.0011, "step": 178, "tokens/total": 5754384, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 80774} +{"epoch": 0.69921875, "grad_norm": 0.3986147344112396, "learning_rate": 7.980420642489674e-05, "loss": 0.0038069630973041058, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00381, "step": 179, "tokens/total": 5789344, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 81247} +{"epoch": 0.703125, "grad_norm": 0.3296111524105072, "learning_rate": 7.95614819466576e-05, "loss": 0.0037569988053292036, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00376, "step": 180, "tokens/total": 5821816, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 81713} +{"epoch": 0.70703125, "grad_norm": 0.8469885587692261, "learning_rate": 7.931773536489872e-05, "loss": 0.015289905481040478, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.01541, "step": 181, "tokens/total": 5854168, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 82158} +{"epoch": 0.7109375, "grad_norm": 0.9009194374084473, "learning_rate": 7.907297682291035e-05, "loss": 0.019129207357764244, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01931, "step": 182, "tokens/total": 5886064, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 82621} +{"epoch": 0.71484375, "grad_norm": 0.7975912690162659, "learning_rate": 7.882721650609442e-05, "loss": 0.007377441972494125, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.0074, "step": 183, "tokens/total": 5921648, "tokens/train_per_sec_per_gpu": 6.66, "tokens/trainable": 83122} +{"epoch": 0.71875, "grad_norm": 0.3530406653881073, "learning_rate": 7.85804646415409e-05, "loss": 0.0054792077280581, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.00549, "step": 184, "tokens/total": 5954952, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 83578} +{"epoch": 0.72265625, "grad_norm": 0.36183103919029236, "learning_rate": 7.833273149760207e-05, "loss": 0.0027030569035559893, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00271, "step": 185, "tokens/total": 5985432, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 84048} +{"epoch": 0.7265625, "grad_norm": 0.3589716851711273, "learning_rate": 7.808402738346527e-05, "loss": 0.00445021316409111, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00446, "step": 186, "tokens/total": 6020664, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 84497} +{"epoch": 0.73046875, "grad_norm": 1.5725886821746826, "learning_rate": 7.783436264872382e-05, "loss": 0.007435370236635208, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00746, "step": 187, "tokens/total": 6055024, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 84983} +{"epoch": 0.734375, "grad_norm": 0.17714256048202515, "learning_rate": 7.758374768294647e-05, "loss": 0.005020580720156431, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00503, "step": 188, "tokens/total": 6088296, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 85462} +{"epoch": 0.73828125, "grad_norm": 0.12316055595874786, "learning_rate": 7.733219291524489e-05, "loss": 0.0020410376600921154, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00204, "step": 189, "tokens/total": 6119840, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 85892} +{"epoch": 0.7421875, "grad_norm": 0.4456588625907898, "learning_rate": 7.707970881383977e-05, "loss": 0.014000286348164082, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.0141, "step": 190, "tokens/total": 6149880, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 86341} +{"epoch": 0.74609375, "grad_norm": 0.6144921779632568, "learning_rate": 7.682630588562518e-05, "loss": 0.004877636209130287, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00489, "step": 191, "tokens/total": 6182904, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 86772} +{"epoch": 0.75, "grad_norm": 0.5412614345550537, "learning_rate": 7.657199467573129e-05, "loss": 0.004977330565452576, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00499, "step": 192, "tokens/total": 6214144, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 87244} +{"epoch": 0.75390625, "grad_norm": 0.428414911031723, "learning_rate": 7.631678576708561e-05, "loss": 0.008843593299388885, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00888, "step": 193, "tokens/total": 6247072, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 87736} +{"epoch": 0.7578125, "grad_norm": 0.03651491180062294, "learning_rate": 7.606068977997255e-05, "loss": 0.00036844704300165176, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00037, "step": 194, "tokens/total": 6281112, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 88214} +{"epoch": 0.76171875, "grad_norm": 0.12874187529087067, "learning_rate": 7.580371737159148e-05, "loss": 0.002022216096520424, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00202, "step": 195, "tokens/total": 6310200, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 88625} +{"epoch": 0.765625, "grad_norm": 0.23301991820335388, "learning_rate": 7.554587923561324e-05, "loss": 0.0020267118234187365, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00203, "step": 196, "tokens/total": 6341792, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 89063} +{"epoch": 0.76953125, "grad_norm": 0.28741180896759033, "learning_rate": 7.528718610173511e-05, "loss": 0.003422651207074523, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.00343, "step": 197, "tokens/total": 6374040, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 89484} +{"epoch": 0.7734375, "grad_norm": 0.38930070400238037, "learning_rate": 7.502764873523431e-05, "loss": 0.0033737332560122013, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00338, "step": 198, "tokens/total": 6405552, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 89922} +{"epoch": 0.77734375, "grad_norm": 1.0397601127624512, "learning_rate": 7.476727793652011e-05, "loss": 0.012617183849215508, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.0127, "step": 199, "tokens/total": 6436016, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 90401} +{"epoch": 0.78125, "grad_norm": 0.4523245394229889, "learning_rate": 7.450608454068415e-05, "loss": 0.013471061363816261, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.01356, "step": 200, "tokens/total": 6468656, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 90842} +{"epoch": 0.78515625, "grad_norm": 0.5528374314308167, "learning_rate": 7.424407941704987e-05, "loss": 0.010776042938232422, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.01083, "step": 201, "tokens/total": 6502728, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 91343} +{"epoch": 0.7890625, "grad_norm": 0.743108868598938, "learning_rate": 7.398127346871986e-05, "loss": 0.0041315192356705666, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00414, "step": 202, "tokens/total": 6535616, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 91798} +{"epoch": 0.79296875, "grad_norm": 0.2519350051879883, "learning_rate": 7.371767763212238e-05, "loss": 0.0030212763231247663, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00303, "step": 203, "tokens/total": 6567088, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 92251} +{"epoch": 0.796875, "grad_norm": 0.0434066504240036, "learning_rate": 7.345330287655617e-05, "loss": 0.0005230801762081683, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00052, "step": 204, "tokens/total": 6600808, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 92714} +{"epoch": 0.80078125, "grad_norm": 0.06735341995954514, "learning_rate": 7.31881602037339e-05, "loss": 0.0009063343750312924, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.00091, "step": 205, "tokens/total": 6631384, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 93154} +{"epoch": 0.8046875, "grad_norm": 0.2098350077867508, "learning_rate": 7.29222606473245e-05, "loss": 0.002889902563765645, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00289, "step": 206, "tokens/total": 6661888, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 93599} +{"epoch": 0.80859375, "grad_norm": 0.13718517124652863, "learning_rate": 7.265561527249383e-05, "loss": 0.0018210154958069324, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00182, "step": 207, "tokens/total": 6693456, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 94036} +{"epoch": 0.8125, "grad_norm": 0.05283189192414284, "learning_rate": 7.238823517544436e-05, "loss": 0.0006476618582382798, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00065, "step": 208, "tokens/total": 6727840, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 94506} +{"epoch": 0.81640625, "grad_norm": 0.3088109791278839, "learning_rate": 7.212013148295333e-05, "loss": 0.0029202366713434458, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00292, "step": 209, "tokens/total": 6761704, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 94988} +{"epoch": 0.8203125, "grad_norm": 0.10233650356531143, "learning_rate": 7.185131535190975e-05, "loss": 0.0009693956817500293, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00097, "step": 210, "tokens/total": 6792648, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 95497} +{"epoch": 0.82421875, "grad_norm": 0.16503094136714935, "learning_rate": 7.158179796885005e-05, "loss": 0.0008063484565354884, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00081, "step": 211, "tokens/total": 6824424, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 95948} +{"epoch": 0.828125, "grad_norm": 0.09622856974601746, "learning_rate": 7.131159054949273e-05, "loss": 0.0004503681557253003, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00045, "step": 212, "tokens/total": 6861216, "tokens/train_per_sec_per_gpu": 6.99, "tokens/trainable": 96422} +{"epoch": 0.83203125, "grad_norm": 0.286601185798645, "learning_rate": 7.104070433827139e-05, "loss": 0.003160464810207486, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00317, "step": 213, "tokens/total": 6892704, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 96821} +{"epoch": 0.8359375, "grad_norm": 0.013987678103148937, "learning_rate": 7.076915060786705e-05, "loss": 0.0001741661544656381, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.00017, "step": 214, "tokens/total": 6924816, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 97263} +{"epoch": 0.83984375, "grad_norm": 0.3609069287776947, "learning_rate": 7.049694065873882e-05, "loss": 0.0028197309002280235, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00282, "step": 215, "tokens/total": 6957536, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 97720} +{"epoch": 0.84375, "grad_norm": 0.6536890268325806, "learning_rate": 7.022408581865382e-05, "loss": 0.009858119301497936, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00991, "step": 216, "tokens/total": 6991448, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 98194} +{"epoch": 0.84765625, "grad_norm": 0.2156648188829422, "learning_rate": 6.99505974422157e-05, "loss": 0.001694964710623026, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.0017, "step": 217, "tokens/total": 7023712, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 98667} +{"epoch": 0.8515625, "grad_norm": 0.06207988038659096, "learning_rate": 6.967648691039213e-05, "loss": 0.0004238212713971734, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00042, "step": 218, "tokens/total": 7055040, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 99100} +{"epoch": 0.85546875, "grad_norm": 0.017087599262595177, "learning_rate": 6.940176563004123e-05, "loss": 0.00013545510591939092, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00014, "step": 219, "tokens/total": 7084944, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 99532} +{"epoch": 0.859375, "grad_norm": 0.8447832465171814, "learning_rate": 6.912644503343682e-05, "loss": 0.003018199000507593, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00302, "step": 220, "tokens/total": 7119264, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 99970} +{"epoch": 0.86328125, "grad_norm": 0.03743588551878929, "learning_rate": 6.885053657779273e-05, "loss": 0.0003198781341779977, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.00032, "step": 221, "tokens/total": 7150848, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 100442} +{"epoch": 0.8671875, "grad_norm": 0.21262112259864807, "learning_rate": 6.857405174478604e-05, "loss": 0.0016710422933101654, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00167, "step": 222, "tokens/total": 7185176, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 100895} +{"epoch": 0.87109375, "grad_norm": 0.1364479660987854, "learning_rate": 6.82970020400792e-05, "loss": 0.0007745149196125567, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00077, "step": 223, "tokens/total": 7219024, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 101342} +{"epoch": 0.875, "grad_norm": 0.8772441744804382, "learning_rate": 6.801939899284132e-05, "loss": 0.005091129336506128, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.0051, "step": 224, "tokens/total": 7253816, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 101848} +{"epoch": 0.87890625, "grad_norm": 0.19143427908420563, "learning_rate": 6.774125415526827e-05, "loss": 0.0008112930227071047, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00081, "step": 225, "tokens/total": 7286344, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 102319} +{"epoch": 0.8828125, "grad_norm": 0.018793171271681786, "learning_rate": 6.746257910210214e-05, "loss": 0.00014430152077693492, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00014, "step": 226, "tokens/total": 7317952, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 102769} +{"epoch": 0.88671875, "grad_norm": 0.23914186656475067, "learning_rate": 6.718338543014937e-05, "loss": 0.0013021811610087752, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.0013, "step": 227, "tokens/total": 7347336, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 103219} +{"epoch": 0.890625, "grad_norm": 1.3783302307128906, "learning_rate": 6.69036847577983e-05, "loss": 0.013492417521774769, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.01358, "step": 228, "tokens/total": 7377592, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 103675} +{"epoch": 0.89453125, "grad_norm": 0.4617903530597687, "learning_rate": 6.662348872453553e-05, "loss": 0.011825963854789734, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.0119, "step": 229, "tokens/total": 7410184, "tokens/train_per_sec_per_gpu": 10.34, "tokens/trainable": 104136} +{"epoch": 0.8984375, "grad_norm": 0.05136260390281677, "learning_rate": 6.63428089904618e-05, "loss": 4.566641291603446e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00005, "step": 230, "tokens/total": 7443984, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 104588} +{"epoch": 0.90234375, "grad_norm": 0.027256624773144722, "learning_rate": 6.60616572358065e-05, "loss": 0.0001373111008433625, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00014, "step": 231, "tokens/total": 7472480, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 104988} +{"epoch": 0.90625, "grad_norm": 0.06914074718952179, "learning_rate": 6.578004516044172e-05, "loss": 0.0005104828160256147, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00051, "step": 232, "tokens/total": 7505992, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 105455} +{"epoch": 0.91015625, "grad_norm": 0.1842007040977478, "learning_rate": 6.549798448339548e-05, "loss": 0.0010936062317341566, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00109, "step": 233, "tokens/total": 7541168, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 105918} +{"epoch": 0.9140625, "grad_norm": 0.3258870244026184, "learning_rate": 6.521548694236384e-05, "loss": 0.0027313653845340014, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00274, "step": 234, "tokens/total": 7574208, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 106355} +{"epoch": 0.91796875, "grad_norm": 0.3053254783153534, "learning_rate": 6.493256429322259e-05, "loss": 0.00306315696798265, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00307, "step": 235, "tokens/total": 7607016, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 106786} +{"epoch": 0.921875, "grad_norm": 0.5932170152664185, "learning_rate": 6.464922830953799e-05, "loss": 0.005712250247597694, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00573, "step": 236, "tokens/total": 7641744, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 107219} +{"epoch": 0.92578125, "grad_norm": 0.6854791641235352, "learning_rate": 6.436549078207688e-05, "loss": 0.011381586082279682, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.01145, "step": 237, "tokens/total": 7673096, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 107679} +{"epoch": 0.9296875, "grad_norm": 0.07286003232002258, "learning_rate": 6.408136351831592e-05, "loss": 0.00040664069820195436, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00041, "step": 238, "tokens/total": 7705344, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 108119} +{"epoch": 0.93359375, "grad_norm": 0.05375217646360397, "learning_rate": 6.379685834195036e-05, "loss": 0.0003101520414929837, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00031, "step": 239, "tokens/total": 7740240, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 108626} +{"epoch": 0.9375, "grad_norm": 0.005354170221835375, "learning_rate": 6.351198709240186e-05, "loss": 7.824816566426307e-05, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00008, "step": 240, "tokens/total": 7775392, "tokens/train_per_sec_per_gpu": 6.98, "tokens/trainable": 109079} +{"epoch": 0.94140625, "grad_norm": 0.08411772549152374, "learning_rate": 6.32267616243259e-05, "loss": 0.0007143376278690994, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00071, "step": 241, "tokens/total": 7807296, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 109542} +{"epoch": 0.9453125, "grad_norm": 0.5257716178894043, "learning_rate": 6.294119380711849e-05, "loss": 0.005709347780793905, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00573, "step": 242, "tokens/total": 7838200, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 109969} +{"epoch": 0.94921875, "grad_norm": 0.48562541604042053, "learning_rate": 6.265529552442209e-05, "loss": 0.0053244177252054214, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00534, "step": 243, "tokens/total": 7870936, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 110431} +{"epoch": 0.953125, "grad_norm": 0.1784723848104477, "learning_rate": 6.236907867363127e-05, "loss": 0.0019081698264926672, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00191, "step": 244, "tokens/total": 7902512, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 110877} +{"epoch": 0.95703125, "grad_norm": 0.2067026048898697, "learning_rate": 6.208255516539749e-05, "loss": 0.0013889011461287737, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00139, "step": 245, "tokens/total": 7933224, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 111350} +{"epoch": 0.9609375, "grad_norm": 0.39953023195266724, "learning_rate": 6.179573692313344e-05, "loss": 0.002126223174855113, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00213, "step": 246, "tokens/total": 7968456, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 111798} +{"epoch": 0.96484375, "grad_norm": 0.14071869850158691, "learning_rate": 6.150863588251694e-05, "loss": 0.0019136472837999463, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00192, "step": 247, "tokens/total": 8003592, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 112264} +{"epoch": 0.96875, "grad_norm": 0.28847819566726685, "learning_rate": 6.122126399099419e-05, "loss": 0.0037194225005805492, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00373, "step": 248, "tokens/total": 8040048, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 112772} +{"epoch": 0.97265625, "grad_norm": 0.7026265859603882, "learning_rate": 6.0933633207282615e-05, "loss": 0.018308693543076515, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.01848, "step": 249, "tokens/total": 8074000, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 113278} +{"epoch": 0.9765625, "grad_norm": 0.42578035593032837, "learning_rate": 6.064575550087316e-05, "loss": 0.005487287417054176, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.0055, "step": 250, "tokens/total": 8108304, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 113716} +{"epoch": 0.98046875, "grad_norm": 0.8372628688812256, "learning_rate": 6.0357642851532245e-05, "loss": 0.013648697175085545, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.01374, "step": 251, "tokens/total": 8143464, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 114207} +{"epoch": 0.984375, "grad_norm": 0.2852499485015869, "learning_rate": 6.0069307248803294e-05, "loss": 0.002536201849579811, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00254, "step": 252, "tokens/total": 8175576, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 114628} +{"epoch": 0.98828125, "grad_norm": 0.052074432373046875, "learning_rate": 5.9780760691507635e-05, "loss": 0.0004207251768093556, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00042, "step": 253, "tokens/total": 8205528, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 115060} +{"epoch": 0.9921875, "grad_norm": 0.07346906512975693, "learning_rate": 5.9492015187245334e-05, "loss": 0.0006630492862313986, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00066, "step": 254, "tokens/total": 8239424, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 115514} +{"epoch": 0.99609375, "grad_norm": 0.5899599194526672, "learning_rate": 5.920308275189541e-05, "loss": 0.022639451548457146, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.0229, "step": 255, "tokens/total": 8270392, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 115915} +{"epoch": 1.0, "grad_norm": 1.768707036972046, "learning_rate": 5.8913975409115874e-05, "loss": 0.003653602907434106, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00366, "step": 256, "tokens/total": 8302096, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 116378} +{"epoch": 1.00390625, "grad_norm": 0.24275290966033936, "learning_rate": 5.8624705189843395e-05, "loss": 0.003764007007703185, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00377, "step": 257, "tokens/total": 8337064, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 116865} +{"epoch": 1.0078125, "grad_norm": 0.17367348074913025, "learning_rate": 5.833528413179249e-05, "loss": 0.003112316131591797, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00312, "step": 258, "tokens/total": 8372840, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 117363} +{"epoch": 1.01171875, "grad_norm": 0.15349526703357697, "learning_rate": 5.80457242789548e-05, "loss": 0.00437828479334712, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00439, "step": 259, "tokens/total": 8409728, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 117860} +{"epoch": 1.015625, "grad_norm": 0.3750828504562378, "learning_rate": 5.77560376810977e-05, "loss": 0.0051689594984054565, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.00518, "step": 260, "tokens/total": 8442296, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 118324} +{"epoch": 1.01953125, "grad_norm": 0.2775129973888397, "learning_rate": 5.7466236393263005e-05, "loss": 0.009311975911259651, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00936, "step": 261, "tokens/total": 8474568, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 118770} +{"epoch": 1.0234375, "grad_norm": 0.30498477816581726, "learning_rate": 5.717633247526522e-05, "loss": 0.005316205322742462, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00533, "step": 262, "tokens/total": 8505520, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 119206} +{"epoch": 1.02734375, "grad_norm": 0.09203324466943741, "learning_rate": 5.688633799118971e-05, "loss": 0.0013929936103522778, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00139, "step": 263, "tokens/total": 8535064, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 119610} +{"epoch": 1.03125, "grad_norm": 0.11789131909608841, "learning_rate": 5.659626500889066e-05, "loss": 0.0026245221961289644, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00263, "step": 264, "tokens/total": 8563600, "tokens/train_per_sec_per_gpu": 6.92, "tokens/trainable": 120080} +{"epoch": 1.03515625, "grad_norm": 0.055188946425914764, "learning_rate": 5.6306125599488905e-05, "loss": 0.000775269465520978, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00078, "step": 265, "tokens/total": 8592648, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 120519} +{"epoch": 1.0390625, "grad_norm": 0.22421564161777496, "learning_rate": 5.601593183686955e-05, "loss": 0.0043310485780239105, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00434, "step": 266, "tokens/total": 8627328, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 120995} +{"epoch": 1.04296875, "grad_norm": 0.48569706082344055, "learning_rate": 5.572569579717961e-05, "loss": 0.004095292184501886, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.0041, "step": 267, "tokens/total": 8659760, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 121459} +{"epoch": 1.046875, "grad_norm": 0.03197921812534332, "learning_rate": 5.543542955832538e-05, "loss": 0.000522589951287955, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00052, "step": 268, "tokens/total": 8696968, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 121902} +{"epoch": 1.05078125, "grad_norm": 0.15078449249267578, "learning_rate": 5.514514519946986e-05, "loss": 0.002177140675485134, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00218, "step": 269, "tokens/total": 8735392, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 122399} +{"epoch": 1.0546875, "grad_norm": 0.10828326642513275, "learning_rate": 5.485485480053015e-05, "loss": 0.001007760874927044, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00101, "step": 270, "tokens/total": 8769840, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 122804} +{"epoch": 1.05859375, "grad_norm": 0.006682696286588907, "learning_rate": 5.4564570441674645e-05, "loss": 9.037908603204414e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00009, "step": 271, "tokens/total": 8800648, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 123244} +{"epoch": 1.0625, "grad_norm": 0.05188708379864693, "learning_rate": 5.42743042028204e-05, "loss": 0.0009123965865001082, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00091, "step": 272, "tokens/total": 8832480, "tokens/train_per_sec_per_gpu": 6.68, "tokens/trainable": 123683} +{"epoch": 1.06640625, "grad_norm": 0.35338518023490906, "learning_rate": 5.3984068163130464e-05, "loss": 0.008171942085027695, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00821, "step": 273, "tokens/total": 8864840, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 124153} +{"epoch": 1.0703125, "grad_norm": 0.3545856475830078, "learning_rate": 5.369387440051111e-05, "loss": 0.004928117152303457, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00494, "step": 274, "tokens/total": 8897272, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 124616} +{"epoch": 1.07421875, "grad_norm": 0.10634787380695343, "learning_rate": 5.340373499110935e-05, "loss": 0.00023738184245303273, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00024, "step": 275, "tokens/total": 8930688, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 125109} +{"epoch": 1.078125, "grad_norm": 0.06479057669639587, "learning_rate": 5.3113662008810304e-05, "loss": 0.000362014485290274, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00036, "step": 276, "tokens/total": 8967216, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 125591} +{"epoch": 1.08203125, "grad_norm": 0.2086368352174759, "learning_rate": 5.282366752473479e-05, "loss": 0.000866413174662739, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00087, "step": 277, "tokens/total": 9001368, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 126065} +{"epoch": 1.0859375, "grad_norm": 0.004373433534055948, "learning_rate": 5.2533763606737005e-05, "loss": 0.00010892403224715963, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00011, "step": 278, "tokens/total": 9036808, "tokens/train_per_sec_per_gpu": 6.78, "tokens/trainable": 126480} +{"epoch": 1.08984375, "grad_norm": 0.07166310399770737, "learning_rate": 5.224396231890232e-05, "loss": 0.00010335495608160272, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.0001, "step": 279, "tokens/total": 9067840, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 126929} +{"epoch": 1.09375, "grad_norm": 0.005019551143050194, "learning_rate": 5.195427572104522e-05, "loss": 0.0001028036349453032, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.0001, "step": 280, "tokens/total": 9098936, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 127366} +{"epoch": 1.09765625, "grad_norm": 0.006347843445837498, "learning_rate": 5.166471586820751e-05, "loss": 6.592126737814397e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00007, "step": 281, "tokens/total": 9133264, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 127831} +{"epoch": 1.1015625, "grad_norm": 0.05428042262792587, "learning_rate": 5.1375294810156615e-05, "loss": 0.00018556487339083105, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00019, "step": 282, "tokens/total": 9165008, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 128278} +{"epoch": 1.10546875, "grad_norm": 0.6000857353210449, "learning_rate": 5.1086024590884144e-05, "loss": 0.006738418713212013, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00676, "step": 283, "tokens/total": 9197952, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 128727} +{"epoch": 1.109375, "grad_norm": 0.020822949707508087, "learning_rate": 5.079691724810461e-05, "loss": 0.0002721439232118428, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00027, "step": 284, "tokens/total": 9232576, "tokens/train_per_sec_per_gpu": 6.99, "tokens/trainable": 129162} +{"epoch": 1.11328125, "grad_norm": 0.017922300845384598, "learning_rate": 5.0507984812754684e-05, "loss": 0.0001169967872556299, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00012, "step": 285, "tokens/total": 9264568, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 129659} +{"epoch": 1.1171875, "grad_norm": 0.05367863178253174, "learning_rate": 5.021923930849237e-05, "loss": 0.00021398899843916297, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00021, "step": 286, "tokens/total": 9296544, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 130145} +{"epoch": 1.12109375, "grad_norm": 0.01037662848830223, "learning_rate": 4.99306927511967e-05, "loss": 6.697954813716933e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00007, "step": 287, "tokens/total": 9327936, "tokens/train_per_sec_per_gpu": 10.25, "tokens/trainable": 130585} +{"epoch": 1.125, "grad_norm": 0.13685627281665802, "learning_rate": 4.964235714846775e-05, "loss": 0.0003446003538556397, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00034, "step": 288, "tokens/total": 9357984, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 131005} +{"epoch": 1.12890625, "grad_norm": 0.006828793324530125, "learning_rate": 4.9354244499126866e-05, "loss": 3.923854819731787e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00004, "step": 289, "tokens/total": 9388664, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 131471} +{"epoch": 1.1328125, "grad_norm": 0.014670468866825104, "learning_rate": 4.90663667927174e-05, "loss": 0.00015022067236714065, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00015, "step": 290, "tokens/total": 9420504, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 131920} +{"epoch": 1.13671875, "grad_norm": 0.01718328893184662, "learning_rate": 4.877873600900581e-05, "loss": 0.00012130187678849325, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00012, "step": 291, "tokens/total": 9452664, "tokens/train_per_sec_per_gpu": 6.19, "tokens/trainable": 132379} +{"epoch": 1.140625, "grad_norm": 0.15917639434337616, "learning_rate": 4.849136411748306e-05, "loss": 0.0009785088477656245, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00098, "step": 292, "tokens/total": 9488896, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 132851} +{"epoch": 1.14453125, "grad_norm": 0.13794057071208954, "learning_rate": 4.8204263076866574e-05, "loss": 0.0006146880914457142, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00061, "step": 293, "tokens/total": 9522112, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 133351} +{"epoch": 1.1484375, "grad_norm": 0.033143311738967896, "learning_rate": 4.791744483460251e-05, "loss": 0.00013083021622151136, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00013, "step": 294, "tokens/total": 9555632, "tokens/train_per_sec_per_gpu": 6.48, "tokens/trainable": 133784} +{"epoch": 1.15234375, "grad_norm": 0.07355429977178574, "learning_rate": 4.7630921326368736e-05, "loss": 0.00031295642838813365, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00031, "step": 295, "tokens/total": 9590904, "tokens/train_per_sec_per_gpu": 6.32, "tokens/trainable": 134209} +{"epoch": 1.15625, "grad_norm": 0.6520754098892212, "learning_rate": 4.7344704475577916e-05, "loss": 0.0038666909094899893, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00387, "step": 296, "tokens/total": 9620288, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 134645} +{"epoch": 1.16015625, "grad_norm": 0.08781830966472626, "learning_rate": 4.705880619288153e-05, "loss": 0.0003635732573457062, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00036, "step": 297, "tokens/total": 9651984, "tokens/train_per_sec_per_gpu": 6.97, "tokens/trainable": 135047} +{"epoch": 1.1640625, "grad_norm": 0.10611757636070251, "learning_rate": 4.677323837567412e-05, "loss": 0.00032259203726425767, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00032, "step": 298, "tokens/total": 9681368, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 135474} +{"epoch": 1.16796875, "grad_norm": 0.0030452292412519455, "learning_rate": 4.6488012907598146e-05, "loss": 2.0680221496149898e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00002, "step": 299, "tokens/total": 9712872, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 135948} +{"epoch": 1.171875, "grad_norm": 0.04685891792178154, "learning_rate": 4.620314165804964e-05, "loss": 0.0003360530245117843, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00034, "step": 300, "tokens/total": 9745184, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 136421} +{"epoch": 1.17578125, "grad_norm": 1.6633074283599854, "learning_rate": 4.591863648168407e-05, "loss": 0.006112103816121817, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00613, "step": 301, "tokens/total": 9778928, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 136886} +{"epoch": 1.1796875, "grad_norm": 0.35164743661880493, "learning_rate": 4.5634509217923135e-05, "loss": 0.006235866341739893, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00626, "step": 302, "tokens/total": 9810752, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 137321} +{"epoch": 1.18359375, "grad_norm": 0.005133600905537605, "learning_rate": 4.535077169046201e-05, "loss": 2.501086964912247e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00003, "step": 303, "tokens/total": 9843672, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 137758} +{"epoch": 1.1875, "grad_norm": 0.3943149149417877, "learning_rate": 4.506743570677743e-05, "loss": 0.0024445236194878817, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00245, "step": 304, "tokens/total": 9873752, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 138190} +{"epoch": 1.19140625, "grad_norm": 0.007639686111360788, "learning_rate": 4.478451305763618e-05, "loss": 5.038548988522962e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00005, "step": 305, "tokens/total": 9906272, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 138650} +{"epoch": 1.1953125, "grad_norm": 0.0014392484445124865, "learning_rate": 4.450201551660454e-05, "loss": 1.4647461284766905e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00001, "step": 306, "tokens/total": 9936296, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 139082} +{"epoch": 1.19921875, "grad_norm": 0.42268702387809753, "learning_rate": 4.4219954839558276e-05, "loss": 0.002390607027336955, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00239, "step": 307, "tokens/total": 9967720, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 139523} +{"epoch": 1.203125, "grad_norm": 0.010667166672647, "learning_rate": 4.393834276419352e-05, "loss": 7.226929301396012e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.00007, "step": 308, "tokens/total": 9997176, "tokens/train_per_sec_per_gpu": 10.6, "tokens/trainable": 140010} +{"epoch": 1.20703125, "grad_norm": 0.02388717047870159, "learning_rate": 4.36571910095382e-05, "loss": 7.863022619858384e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00008, "step": 309, "tokens/total": 10031872, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 140454} +{"epoch": 1.2109375, "grad_norm": 0.16001495718955994, "learning_rate": 4.337651127546448e-05, "loss": 0.0005229230737313628, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00052, "step": 310, "tokens/total": 10065240, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 140905} +{"epoch": 1.21484375, "grad_norm": 0.432526558637619, "learning_rate": 4.3096315242201736e-05, "loss": 0.010566060431301594, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.01062, "step": 311, "tokens/total": 10096640, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 141339} +{"epoch": 1.21875, "grad_norm": 0.004630552139133215, "learning_rate": 4.2816614569850635e-05, "loss": 5.54832695343066e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00006, "step": 312, "tokens/total": 10130192, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 141778} +{"epoch": 1.22265625, "grad_norm": 0.025235069915652275, "learning_rate": 4.2537420897897864e-05, "loss": 0.00016743309970479459, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00017, "step": 313, "tokens/total": 10164248, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 142255} +{"epoch": 1.2265625, "grad_norm": 0.37162312865257263, "learning_rate": 4.225874584473174e-05, "loss": 0.0018221476348116994, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00182, "step": 314, "tokens/total": 10194664, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 142693} +{"epoch": 1.23046875, "grad_norm": 0.024704892188310623, "learning_rate": 4.19806010071587e-05, "loss": 0.00023171669454313815, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00023, "step": 315, "tokens/total": 10227768, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 143157} +{"epoch": 1.234375, "grad_norm": 0.005770247429609299, "learning_rate": 4.170299795992081e-05, "loss": 5.781522486358881e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00006, "step": 316, "tokens/total": 10260464, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 143603} +{"epoch": 1.23828125, "grad_norm": 0.46698886156082153, "learning_rate": 4.142594825521398e-05, "loss": 0.0022447870578616858, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00225, "step": 317, "tokens/total": 10297528, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 144084} +{"epoch": 1.2421875, "grad_norm": 0.0007949387654662132, "learning_rate": 4.114946342220728e-05, "loss": 1.6397172657889314e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00002, "step": 318, "tokens/total": 10331448, "tokens/train_per_sec_per_gpu": 6.85, "tokens/trainable": 144532} +{"epoch": 1.24609375, "grad_norm": 0.0040134708397090435, "learning_rate": 4.087355496656321e-05, "loss": 5.885145583306439e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00006, "step": 319, "tokens/total": 10363152, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 145004} +{"epoch": 1.25, "grad_norm": 0.3450433909893036, "learning_rate": 4.05982343699588e-05, "loss": 0.0006297366926446557, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00063, "step": 320, "tokens/total": 10397832, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 145498} +{"epoch": 1.25390625, "grad_norm": 0.009933366440236568, "learning_rate": 4.0323513089607876e-05, "loss": 5.554355448111892e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00006, "step": 321, "tokens/total": 10431416, "tokens/train_per_sec_per_gpu": 7.24, "tokens/trainable": 145965} +{"epoch": 1.2578125, "grad_norm": 0.04992857575416565, "learning_rate": 4.004940255778431e-05, "loss": 0.00016838469309732318, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00017, "step": 322, "tokens/total": 10467384, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 146409} +{"epoch": 1.26171875, "grad_norm": 0.12144682556390762, "learning_rate": 3.977591418134619e-05, "loss": 0.0006065507186576724, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00061, "step": 323, "tokens/total": 10500792, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 146849} +{"epoch": 1.265625, "grad_norm": 0.026350008323788643, "learning_rate": 3.95030593412612e-05, "loss": 0.0001838075404521078, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00018, "step": 324, "tokens/total": 10532592, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 147272} +{"epoch": 1.26953125, "grad_norm": 0.0031925884541124105, "learning_rate": 3.923084939213296e-05, "loss": 4.0913648263085634e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00004, "step": 325, "tokens/total": 10562392, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 147694} +{"epoch": 1.2734375, "grad_norm": 0.5960577726364136, "learning_rate": 3.895929566172861e-05, "loss": 0.01065824180841446, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.01072, "step": 326, "tokens/total": 10595416, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 148158} +{"epoch": 1.27734375, "grad_norm": 0.012474928982555866, "learning_rate": 3.868840945050728e-05, "loss": 0.0001510022848378867, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00015, "step": 327, "tokens/total": 10630504, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 148590} +{"epoch": 1.28125, "grad_norm": 0.3829227685928345, "learning_rate": 3.841820203114995e-05, "loss": 0.0021044316235929728, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00211, "step": 328, "tokens/total": 10661632, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 149037} +{"epoch": 1.28515625, "grad_norm": 0.038173574954271317, "learning_rate": 3.814868464809027e-05, "loss": 0.00016331940423697233, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00016, "step": 329, "tokens/total": 10691544, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 149439} +{"epoch": 1.2890625, "grad_norm": 0.006043481174856424, "learning_rate": 3.787986851704667e-05, "loss": 8.188884385162964e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00008, "step": 330, "tokens/total": 10724768, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 149869} +{"epoch": 1.29296875, "grad_norm": 0.3262004554271698, "learning_rate": 3.7611764824555654e-05, "loss": 0.002785277785733342, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00279, "step": 331, "tokens/total": 10756008, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 150343} +{"epoch": 1.296875, "grad_norm": 0.11662720143795013, "learning_rate": 3.734438472750619e-05, "loss": 0.0006444333121180534, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00064, "step": 332, "tokens/total": 10793152, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 150814} +{"epoch": 1.30078125, "grad_norm": 0.0019426638027653098, "learning_rate": 3.707773935267552e-05, "loss": 2.2956737666390836e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00002, "step": 333, "tokens/total": 10827624, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 151300} +{"epoch": 1.3046875, "grad_norm": 0.0009356829104945064, "learning_rate": 3.68118397962661e-05, "loss": 1.9032260752283037e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00002, "step": 334, "tokens/total": 10859712, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 151708} +{"epoch": 1.30859375, "grad_norm": 0.0012037890264764428, "learning_rate": 3.654669712344384e-05, "loss": 1.6301937648677267e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00002, "step": 335, "tokens/total": 10888792, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 152164} +{"epoch": 1.3125, "grad_norm": 0.007404594216495752, "learning_rate": 3.628232236787763e-05, "loss": 5.6950459111249074e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00006, "step": 336, "tokens/total": 10922968, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 152603} +{"epoch": 1.31640625, "grad_norm": 0.28917625546455383, "learning_rate": 3.6018726531280144e-05, "loss": 0.0011542326537892222, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00115, "step": 337, "tokens/total": 10958000, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 153094} +{"epoch": 1.3203125, "grad_norm": 0.04606437683105469, "learning_rate": 3.575592058295017e-05, "loss": 0.0002899376268032938, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00029, "step": 338, "tokens/total": 10989872, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 153573} +{"epoch": 1.32421875, "grad_norm": 0.012756210751831532, "learning_rate": 3.549391545931585e-05, "loss": 9.493259858572856e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00009, "step": 339, "tokens/total": 11020784, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 154033} +{"epoch": 1.328125, "grad_norm": 0.04035387560725212, "learning_rate": 3.5232722063479914e-05, "loss": 0.00016576182679273188, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00017, "step": 340, "tokens/total": 11053240, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 154482} +{"epoch": 1.33203125, "grad_norm": 0.0019543934613466263, "learning_rate": 3.49723512647657e-05, "loss": 3.646373806986958e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00004, "step": 341, "tokens/total": 11089040, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 154982} +{"epoch": 1.3359375, "grad_norm": 0.009206002578139305, "learning_rate": 3.471281389826491e-05, "loss": 6.42234954284504e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00006, "step": 342, "tokens/total": 11119672, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 155387} +{"epoch": 1.33984375, "grad_norm": 0.20961177349090576, "learning_rate": 3.4454120764386764e-05, "loss": 0.0015076876152306795, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00151, "step": 343, "tokens/total": 11155040, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 155842} +{"epoch": 1.34375, "grad_norm": 0.6517477035522461, "learning_rate": 3.4196282628408526e-05, "loss": 0.009738093242049217, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00979, "step": 344, "tokens/total": 11185952, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 156287} +{"epoch": 1.34765625, "grad_norm": 0.03230731934309006, "learning_rate": 3.3939310220027456e-05, "loss": 0.0002046791196335107, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.0002, "step": 345, "tokens/total": 11219272, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 156782} +{"epoch": 1.3515625, "grad_norm": 0.02635231241583824, "learning_rate": 3.3683214232914404e-05, "loss": 0.0001432376739103347, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00014, "step": 346, "tokens/total": 11253424, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 157255} +{"epoch": 1.35546875, "grad_norm": 0.004715971648693085, "learning_rate": 3.342800532426873e-05, "loss": 3.624531382229179e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00004, "step": 347, "tokens/total": 11286216, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 157721} +{"epoch": 1.359375, "grad_norm": 0.20174816250801086, "learning_rate": 3.317369411437484e-05, "loss": 0.0031131517607718706, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00312, "step": 348, "tokens/total": 11319480, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 158189} +{"epoch": 1.36328125, "grad_norm": 0.007512213662266731, "learning_rate": 3.292029118616024e-05, "loss": 0.0001276958646485582, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00013, "step": 349, "tokens/total": 11350472, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 158612} +{"epoch": 1.3671875, "grad_norm": 0.8888293504714966, "learning_rate": 3.266780708475511e-05, "loss": 0.006944428198039532, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00697, "step": 350, "tokens/total": 11383088, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 159090} +{"epoch": 1.37109375, "grad_norm": 0.015433511696755886, "learning_rate": 3.241625231705354e-05, "loss": 7.357387221418321e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00007, "step": 351, "tokens/total": 11418392, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 159549} +{"epoch": 1.375, "grad_norm": 0.0018381632398813963, "learning_rate": 3.216563735127618e-05, "loss": 2.3467553546652198e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00002, "step": 352, "tokens/total": 11449784, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 160026} +{"epoch": 1.37890625, "grad_norm": 0.006945513654500246, "learning_rate": 3.191597261653475e-05, "loss": 3.465404734015465e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00003, "step": 353, "tokens/total": 11484240, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 160487} +{"epoch": 1.3828125, "grad_norm": 0.032304923981428146, "learning_rate": 3.166726850239794e-05, "loss": 0.00014519527030643076, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00015, "step": 354, "tokens/total": 11513360, "tokens/train_per_sec_per_gpu": 10.81, "tokens/trainable": 160935} +{"epoch": 1.38671875, "grad_norm": 0.009348016232252121, "learning_rate": 3.141953535845912e-05, "loss": 4.393159906612709e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00004, "step": 355, "tokens/total": 11543112, "tokens/train_per_sec_per_gpu": 10.08, "tokens/trainable": 161385} +{"epoch": 1.390625, "grad_norm": 0.002148550236597657, "learning_rate": 3.11727834939056e-05, "loss": 3.453902172623202e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00003, "step": 356, "tokens/total": 11573464, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 161849} +{"epoch": 1.39453125, "grad_norm": 0.19447878003120422, "learning_rate": 3.092702317708967e-05, "loss": 0.0021217791363596916, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00212, "step": 357, "tokens/total": 11604896, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 162306} +{"epoch": 1.3984375, "grad_norm": 0.007358372211456299, "learning_rate": 3.0682264635101276e-05, "loss": 5.700471228919923e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00006, "step": 358, "tokens/total": 11636368, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 162759} +{"epoch": 1.40234375, "grad_norm": 0.004709967412054539, "learning_rate": 3.0438518053342407e-05, "loss": 4.352873656898737e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00004, "step": 359, "tokens/total": 11670216, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 163230} +{"epoch": 1.40625, "grad_norm": 0.005764028988778591, "learning_rate": 3.0195793575103266e-05, "loss": 6.834438681835309e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00007, "step": 360, "tokens/total": 11707816, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 163694} +{"epoch": 1.41015625, "grad_norm": 2.162550687789917, "learning_rate": 2.9954101301140146e-05, "loss": 0.003165688831359148, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00317, "step": 361, "tokens/total": 11742584, "tokens/train_per_sec_per_gpu": 6.8, "tokens/trainable": 164144} +{"epoch": 1.4140625, "grad_norm": 0.05700010433793068, "learning_rate": 2.9713451289255123e-05, "loss": 0.00031535470043309033, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00032, "step": 362, "tokens/total": 11770384, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 164585} +{"epoch": 1.41796875, "grad_norm": 0.0019377791322767735, "learning_rate": 2.9473853553877484e-05, "loss": 3.515169373713434e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00004, "step": 363, "tokens/total": 11803208, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 165059} +{"epoch": 1.421875, "grad_norm": 0.003441296052187681, "learning_rate": 2.9235318065647e-05, "loss": 3.382346767466515e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00003, "step": 364, "tokens/total": 11840480, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 165519} +{"epoch": 1.42578125, "grad_norm": 0.0025044793728739023, "learning_rate": 2.8997854750998964e-05, "loss": 3.123666101600975e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00003, "step": 365, "tokens/total": 11875544, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 165984} +{"epoch": 1.4296875, "grad_norm": 0.0020933696068823338, "learning_rate": 2.8761473491751258e-05, "loss": 2.549876444390975e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00003, "step": 366, "tokens/total": 11907656, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 166419} +{"epoch": 1.43359375, "grad_norm": 0.007798216305673122, "learning_rate": 2.8526184124692883e-05, "loss": 7.579291559522972e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00008, "step": 367, "tokens/total": 11941904, "tokens/train_per_sec_per_gpu": 6.29, "tokens/trainable": 166857} +{"epoch": 1.4375, "grad_norm": 0.1902092844247818, "learning_rate": 2.829199644117484e-05, "loss": 0.0008692098199389875, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00087, "step": 368, "tokens/total": 11973624, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 167267} +{"epoch": 1.44140625, "grad_norm": 0.0032868352718651295, "learning_rate": 2.8058920186702553e-05, "loss": 3.773558637476526e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00004, "step": 369, "tokens/total": 12010360, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 167740} +{"epoch": 1.4453125, "grad_norm": 0.006895545870065689, "learning_rate": 2.782696506053033e-05, "loss": 6.973161362111568e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00007, "step": 370, "tokens/total": 12044544, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 168172} +{"epoch": 1.44921875, "grad_norm": 0.2893476188182831, "learning_rate": 2.7596140715257824e-05, "loss": 0.0024223425425589085, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00243, "step": 371, "tokens/total": 12075792, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 168637} +{"epoch": 1.453125, "grad_norm": 0.003180361120030284, "learning_rate": 2.7366456756428184e-05, "loss": 2.4921580916270614e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00002, "step": 372, "tokens/total": 12108088, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 169094} +{"epoch": 1.45703125, "grad_norm": 0.05256488174200058, "learning_rate": 2.7137922742128486e-05, "loss": 0.00022829596127849072, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00023, "step": 373, "tokens/total": 12140888, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 169593} +{"epoch": 1.4609375, "grad_norm": 0.6477333903312683, "learning_rate": 2.691054818259188e-05, "loss": 0.00603412464261055, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00605, "step": 374, "tokens/total": 12174560, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 170066} +{"epoch": 1.46484375, "grad_norm": 0.30182522535324097, "learning_rate": 2.6684342539801933e-05, "loss": 0.0009333090856671333, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00093, "step": 375, "tokens/total": 12204848, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 170518} +{"epoch": 1.46875, "grad_norm": 0.4360727369785309, "learning_rate": 2.645931522709877e-05, "loss": 0.0059890057891607285, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00601, "step": 376, "tokens/total": 12238472, "tokens/train_per_sec_per_gpu": 10.56, "tokens/trainable": 170998} +{"epoch": 1.47265625, "grad_norm": 0.30627134442329407, "learning_rate": 2.6235475608787365e-05, "loss": 0.000577284547034651, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00058, "step": 377, "tokens/total": 12271024, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 171459} +{"epoch": 1.4765625, "grad_norm": 0.3660154640674591, "learning_rate": 2.6012832999747916e-05, "loss": 0.0010844548232853413, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00109, "step": 378, "tokens/total": 12301312, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 171905} +{"epoch": 1.48046875, "grad_norm": 0.9876859188079834, "learning_rate": 2.579139666504821e-05, "loss": 0.0046632857993245125, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00467, "step": 379, "tokens/total": 12337072, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 172351} +{"epoch": 1.484375, "grad_norm": 0.192829430103302, "learning_rate": 2.557117581955798e-05, "loss": 0.0011655783746391535, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00117, "step": 380, "tokens/total": 12368000, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 172817} +{"epoch": 1.48828125, "grad_norm": 0.45055463910102844, "learning_rate": 2.5352179627565532e-05, "loss": 0.001872414373792708, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00187, "step": 381, "tokens/total": 12399880, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 173258} +{"epoch": 1.4921875, "grad_norm": 0.004000425338745117, "learning_rate": 2.5134417202396277e-05, "loss": 4.2490406485740095e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00004, "step": 382, "tokens/total": 12432288, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 173665} +{"epoch": 1.49609375, "grad_norm": 0.4134214520454407, "learning_rate": 2.491789760603361e-05, "loss": 0.008748912252485752, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00879, "step": 383, "tokens/total": 12465592, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 174095} +{"epoch": 1.5, "grad_norm": 0.059520427137613297, "learning_rate": 2.4702629848741764e-05, "loss": 0.00011237163562327623, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00011, "step": 384, "tokens/total": 12498200, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 174550} +{"epoch": 1.50390625, "grad_norm": 0.07820552587509155, "learning_rate": 2.4488622888690785e-05, "loss": 0.00043206679401919246, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00043, "step": 385, "tokens/total": 12530280, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 174990} +{"epoch": 1.5078125, "grad_norm": 0.004445604979991913, "learning_rate": 2.427588563158384e-05, "loss": 4.870894190389663e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00005, "step": 386, "tokens/total": 12563360, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 175476} +{"epoch": 1.51171875, "grad_norm": 0.003349193837493658, "learning_rate": 2.406442693028651e-05, "loss": 3.83301630790811e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00004, "step": 387, "tokens/total": 12593032, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 175897} +{"epoch": 1.515625, "grad_norm": 0.0015769985038787127, "learning_rate": 2.3854255584458547e-05, "loss": 1.8005670426646248e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00002, "step": 388, "tokens/total": 12628272, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 176324} +{"epoch": 1.51953125, "grad_norm": 0.0010430088732391596, "learning_rate": 2.3645380340187508e-05, "loss": 1.483819323766511e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00001, "step": 389, "tokens/total": 12661368, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 176805} +{"epoch": 1.5234375, "grad_norm": 0.06765414029359818, "learning_rate": 2.3437809889624914e-05, "loss": 0.00025506390375085175, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.77, "memory/max_allocated (GiB)": 63.77, "ppl": 1.00026, "step": 390, "tokens/total": 12691168, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 177256} +{"epoch": 1.52734375, "grad_norm": 0.0014147567562758923, "learning_rate": 2.3231552870624487e-05, "loss": 3.0233295547077432e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00003, "step": 391, "tokens/total": 12721264, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 177682} +{"epoch": 1.53125, "grad_norm": 0.10742944478988647, "learning_rate": 2.3026617866382657e-05, "loss": 0.0004929557326249778, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00049, "step": 392, "tokens/total": 12753640, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 178179} +{"epoch": 1.53515625, "grad_norm": 0.018440725281834602, "learning_rate": 2.2823013405081507e-05, "loss": 0.00010025114170275629, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.0001, "step": 393, "tokens/total": 12787056, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 178676} +{"epoch": 1.5390625, "grad_norm": 0.009249833412468433, "learning_rate": 2.2620747959533722e-05, "loss": 4.243681178195402e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00004, "step": 394, "tokens/total": 12819312, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 179173} +{"epoch": 1.54296875, "grad_norm": 0.03272626921534538, "learning_rate": 2.2419829946830123e-05, "loss": 0.00022411772806663066, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00022, "step": 395, "tokens/total": 12853376, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 179624} +{"epoch": 1.546875, "grad_norm": 0.43350669741630554, "learning_rate": 2.2220267727989325e-05, "loss": 0.00904288049787283, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00908, "step": 396, "tokens/total": 12882960, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 180078} +{"epoch": 1.55078125, "grad_norm": 0.034521620720624924, "learning_rate": 2.202206960760984e-05, "loss": 0.00036833895137533545, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00037, "step": 397, "tokens/total": 12917016, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 180585} +{"epoch": 1.5546875, "grad_norm": 0.0031559024937450886, "learning_rate": 2.182524383352446e-05, "loss": 6.621505599468946e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00007, "step": 398, "tokens/total": 12950056, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 181039} +{"epoch": 1.55859375, "grad_norm": 1.240646481513977, "learning_rate": 2.1629798596457056e-05, "loss": 0.007756279315799475, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00779, "step": 399, "tokens/total": 12981728, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 181489} +{"epoch": 1.5625, "grad_norm": 0.07410334795713425, "learning_rate": 2.1435742029681725e-05, "loss": 0.00020366013632155955, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.0002, "step": 400, "tokens/total": 13011464, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 181906} +{"epoch": 1.56640625, "grad_norm": 0.0012307228753343225, "learning_rate": 2.124308220868431e-05, "loss": 2.6916210117633455e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00003, "step": 401, "tokens/total": 13044432, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 182361} +{"epoch": 1.5703125, "grad_norm": 0.16271954774856567, "learning_rate": 2.105182715082638e-05, "loss": 0.0014348975382745266, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00144, "step": 402, "tokens/total": 13075608, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 182838} +{"epoch": 1.57421875, "grad_norm": 0.016888247802853584, "learning_rate": 2.0861984815011552e-05, "loss": 0.00013697029498871416, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00014, "step": 403, "tokens/total": 13107352, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 183269} +{"epoch": 1.578125, "grad_norm": 0.01827048696577549, "learning_rate": 2.0673563101354323e-05, "loss": 0.00018277092021889985, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00018, "step": 404, "tokens/total": 13140736, "tokens/train_per_sec_per_gpu": 7.11, "tokens/trainable": 183699} +{"epoch": 1.58203125, "grad_norm": 0.018589043989777565, "learning_rate": 2.0486569850851317e-05, "loss": 0.00016926552052609622, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00017, "step": 405, "tokens/total": 13174040, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 184162} +{"epoch": 1.5859375, "grad_norm": 0.007233355659991503, "learning_rate": 2.0301012845054956e-05, "loss": 8.655490091769025e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00009, "step": 406, "tokens/total": 13207232, "tokens/train_per_sec_per_gpu": 6.49, "tokens/trainable": 184576} +{"epoch": 1.58984375, "grad_norm": 0.021000338718295097, "learning_rate": 2.011689980574966e-05, "loss": 0.00012465151667129248, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00012, "step": 407, "tokens/total": 13241160, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 184995} +{"epoch": 1.59375, "grad_norm": 0.3891313672065735, "learning_rate": 1.993423839463052e-05, "loss": 0.006694347131997347, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00672, "step": 408, "tokens/total": 13273104, "tokens/train_per_sec_per_gpu": 10.26, "tokens/trainable": 185452} +{"epoch": 1.59765625, "grad_norm": 0.0031187976710498333, "learning_rate": 1.975303621298445e-05, "loss": 4.605836875271052e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00005, "step": 409, "tokens/total": 13308136, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 185904} +{"epoch": 1.6015625, "grad_norm": 0.011829393915832043, "learning_rate": 1.957330080137385e-05, "loss": 8.231220999732614e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00008, "step": 410, "tokens/total": 13337816, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 186355} +{"epoch": 1.60546875, "grad_norm": 0.7164475917816162, "learning_rate": 1.9395039639322864e-05, "loss": 0.010398062877357006, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.01045, "step": 411, "tokens/total": 13372576, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 186831} +{"epoch": 1.609375, "grad_norm": 0.011289510875940323, "learning_rate": 1.9218260145006073e-05, "loss": 4.689354318543337e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00005, "step": 412, "tokens/total": 13406664, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 187317} +{"epoch": 1.61328125, "grad_norm": 0.005934373941272497, "learning_rate": 1.904296967493982e-05, "loss": 4.9243972171097994e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00005, "step": 413, "tokens/total": 13438256, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 187773} +{"epoch": 1.6171875, "grad_norm": 0.05501369759440422, "learning_rate": 1.8869175523676064e-05, "loss": 0.0001600108516868204, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00016, "step": 414, "tokens/total": 13472096, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 188268} +{"epoch": 1.62109375, "grad_norm": 0.03403918817639351, "learning_rate": 1.869688492349885e-05, "loss": 0.00040956935845315456, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00041, "step": 415, "tokens/total": 13504192, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 188720} +{"epoch": 1.625, "grad_norm": 0.3002409040927887, "learning_rate": 1.85261050441233e-05, "loss": 0.0017155084060505033, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00172, "step": 416, "tokens/total": 13537392, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 189177} +{"epoch": 1.62890625, "grad_norm": 0.3654189705848694, "learning_rate": 1.8356842992397304e-05, "loss": 0.007346402388066053, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00737, "step": 417, "tokens/total": 13569344, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 189617} +{"epoch": 1.6328125, "grad_norm": 0.070885568857193, "learning_rate": 1.8189105812005714e-05, "loss": 0.0007869168766774237, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00079, "step": 418, "tokens/total": 13602200, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 190028} +{"epoch": 1.63671875, "grad_norm": 0.04553167521953583, "learning_rate": 1.802290048317732e-05, "loss": 0.0003029265208169818, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.0003, "step": 419, "tokens/total": 13632640, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 190445} +{"epoch": 1.640625, "grad_norm": 0.018734117969870567, "learning_rate": 1.785823392239424e-05, "loss": 0.00027618446620181203, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00028, "step": 420, "tokens/total": 13664920, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 190886} +{"epoch": 1.64453125, "grad_norm": 0.012697406113147736, "learning_rate": 1.7695112982104225e-05, "loss": 0.00011166264448547736, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00011, "step": 421, "tokens/total": 13697144, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 191325} +{"epoch": 1.6484375, "grad_norm": 0.011028813198208809, "learning_rate": 1.7533544450435433e-05, "loss": 9.796014637686312e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.0001, "step": 422, "tokens/total": 13724496, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 191758} +{"epoch": 1.65234375, "grad_norm": 0.004870921839028597, "learning_rate": 1.7373535050913946e-05, "loss": 3.3909800549736246e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00003, "step": 423, "tokens/total": 13757880, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 192195} +{"epoch": 1.65625, "grad_norm": 0.4335399866104126, "learning_rate": 1.721509144218405e-05, "loss": 0.004736952017992735, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00475, "step": 424, "tokens/total": 13787648, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 192625} +{"epoch": 1.66015625, "grad_norm": 0.016014931723475456, "learning_rate": 1.705822021773101e-05, "loss": 0.0002482207491993904, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00025, "step": 425, "tokens/total": 13819720, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 193111} +{"epoch": 1.6640625, "grad_norm": 0.03828779608011246, "learning_rate": 1.69029279056068e-05, "loss": 0.0003634061722550541, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00036, "step": 426, "tokens/total": 13854944, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 193551} +{"epoch": 1.66796875, "grad_norm": 0.040779341012239456, "learning_rate": 1.6749220968158415e-05, "loss": 0.00037496042205020785, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00038, "step": 427, "tokens/total": 13887824, "tokens/train_per_sec_per_gpu": 10.19, "tokens/trainable": 194057} +{"epoch": 1.671875, "grad_norm": 0.018650861456990242, "learning_rate": 1.659710580175893e-05, "loss": 0.0002787585253827274, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00028, "step": 428, "tokens/total": 13918192, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 194556} +{"epoch": 1.67578125, "grad_norm": 0.02578047104179859, "learning_rate": 1.644658873654133e-05, "loss": 0.00022593606263399124, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00023, "step": 429, "tokens/total": 13947440, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 194993} +{"epoch": 1.6796875, "grad_norm": 0.004055908881127834, "learning_rate": 1.629767603613508e-05, "loss": 4.997415089746937e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00005, "step": 430, "tokens/total": 13974360, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 195393} +{"epoch": 1.68359375, "grad_norm": 0.04618486016988754, "learning_rate": 1.615037389740547e-05, "loss": 0.00034021420287899673, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00034, "step": 431, "tokens/total": 14006592, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 195837} +{"epoch": 1.6875, "grad_norm": 0.5458316802978516, "learning_rate": 1.600468845019576e-05, "loss": 0.003299737349152565, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00331, "step": 432, "tokens/total": 14040776, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 196317} +{"epoch": 1.69140625, "grad_norm": 0.000998991192318499, "learning_rate": 1.5860625757072092e-05, "loss": 2.1111725800437853e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00002, "step": 433, "tokens/total": 14073040, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 196765} +{"epoch": 1.6953125, "grad_norm": 0.004338617902249098, "learning_rate": 1.571819181307116e-05, "loss": 5.213319673202932e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00005, "step": 434, "tokens/total": 14107624, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 197237} +{"epoch": 1.69921875, "grad_norm": 0.03080410696566105, "learning_rate": 1.557739254545075e-05, "loss": 0.00019341017468832433, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00019, "step": 435, "tokens/total": 14137784, "tokens/train_per_sec_per_gpu": 10.44, "tokens/trainable": 197709} +{"epoch": 1.703125, "grad_norm": 0.03308214619755745, "learning_rate": 1.543823381344311e-05, "loss": 0.0001644391450099647, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00016, "step": 436, "tokens/total": 14167776, "tokens/train_per_sec_per_gpu": 10.57, "tokens/trainable": 198184} +{"epoch": 1.70703125, "grad_norm": 0.17077776789665222, "learning_rate": 1.5300721408011114e-05, "loss": 0.0011717649176716805, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00117, "step": 437, "tokens/total": 14199752, "tokens/train_per_sec_per_gpu": 10.04, "tokens/trainable": 198641} +{"epoch": 1.7109375, "grad_norm": 0.012075735256075859, "learning_rate": 1.5164861051607254e-05, "loss": 6.317975203273818e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.00006, "step": 438, "tokens/total": 14230344, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 199049} +{"epoch": 1.71484375, "grad_norm": 0.04102166369557381, "learning_rate": 1.5030658397935521e-05, "loss": 0.00034020308521576226, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00034, "step": 439, "tokens/total": 14263808, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 199518} +{"epoch": 1.71875, "grad_norm": 0.672312319278717, "learning_rate": 1.4898119031716104e-05, "loss": 0.005485543981194496, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.0055, "step": 440, "tokens/total": 14295864, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 199976} +{"epoch": 1.72265625, "grad_norm": 0.18725284934043884, "learning_rate": 1.476724846845306e-05, "loss": 0.0005613222601823509, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00056, "step": 441, "tokens/total": 14331688, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 200453} +{"epoch": 1.7265625, "grad_norm": 0.01820395141839981, "learning_rate": 1.463805215420471e-05, "loss": 0.0002076293167192489, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00021, "step": 442, "tokens/total": 14364688, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 200923} +{"epoch": 1.73046875, "grad_norm": 0.01812027022242546, "learning_rate": 1.451053546535705e-05, "loss": 0.00013140824739821255, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00013, "step": 443, "tokens/total": 14397832, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 201379} +{"epoch": 1.734375, "grad_norm": 0.26284071803092957, "learning_rate": 1.438470370840001e-05, "loss": 0.003315509995445609, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00332, "step": 444, "tokens/total": 14431864, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 201851} +{"epoch": 1.73828125, "grad_norm": 0.012552417814731598, "learning_rate": 1.4260562119706606e-05, "loss": 0.00014530184853356332, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00015, "step": 445, "tokens/total": 14462152, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 202325} +{"epoch": 1.7421875, "grad_norm": 0.0026239410508424044, "learning_rate": 1.413811586531508e-05, "loss": 4.334312689024955e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00004, "step": 446, "tokens/total": 14493072, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 202766} +{"epoch": 1.74609375, "grad_norm": 0.14921002089977264, "learning_rate": 1.4017370040713884e-05, "loss": 0.0016407425282523036, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00164, "step": 447, "tokens/total": 14527392, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 203215} +{"epoch": 1.75, "grad_norm": 1.5497336387634277, "learning_rate": 1.3898329670629645e-05, "loss": 0.0031319863628596067, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00314, "step": 448, "tokens/total": 14561992, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 203660} +{"epoch": 1.75390625, "grad_norm": 0.003092269878834486, "learning_rate": 1.3780999708818058e-05, "loss": 4.699763667304069e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00005, "step": 449, "tokens/total": 14592056, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 204110} +{"epoch": 1.7578125, "grad_norm": 0.006132656708359718, "learning_rate": 1.3665385037857758e-05, "loss": 7.694100349908695e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00008, "step": 450, "tokens/total": 14622816, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 204562} +{"epoch": 1.76171875, "grad_norm": 0.042260900139808655, "learning_rate": 1.3551490468947126e-05, "loss": 0.0003009928041137755, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.0003, "step": 451, "tokens/total": 14653272, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 205008} +{"epoch": 1.765625, "grad_norm": 0.02249080315232277, "learning_rate": 1.3439320741704075e-05, "loss": 0.00022836893913336098, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00023, "step": 452, "tokens/total": 14683952, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 205432} +{"epoch": 1.76953125, "grad_norm": 0.010306959971785545, "learning_rate": 1.3328880523968808e-05, "loss": 0.0001511423906777054, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00015, "step": 453, "tokens/total": 14716256, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 205895} +{"epoch": 1.7734375, "grad_norm": 0.027949903160333633, "learning_rate": 1.3220174411609587e-05, "loss": 0.0001380652975058183, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00014, "step": 454, "tokens/total": 14749088, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 206334} +{"epoch": 1.77734375, "grad_norm": 0.0025757078547030687, "learning_rate": 1.3113206928331471e-05, "loss": 3.5820077755488455e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00004, "step": 455, "tokens/total": 14777384, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 206796} +{"epoch": 1.78125, "grad_norm": 0.050274502485990524, "learning_rate": 1.300798252548806e-05, "loss": 0.00023890442389529198, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00024, "step": 456, "tokens/total": 14809640, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 207248} +{"epoch": 1.78515625, "grad_norm": 0.34974274039268494, "learning_rate": 1.2904505581896265e-05, "loss": 0.01341838575899601, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01351, "step": 457, "tokens/total": 14843728, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 207721} +{"epoch": 1.7890625, "grad_norm": 0.006988106295466423, "learning_rate": 1.2802780403654082e-05, "loss": 0.00012247887207195163, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00012, "step": 458, "tokens/total": 14876256, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 208168} +{"epoch": 1.79296875, "grad_norm": 0.03125142306089401, "learning_rate": 1.2702811223961408e-05, "loss": 0.0003607344697229564, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00036, "step": 459, "tokens/total": 14907000, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 208629} +{"epoch": 1.796875, "grad_norm": 0.004587366711348295, "learning_rate": 1.2604602202943861e-05, "loss": 8.442281250609085e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00008, "step": 460, "tokens/total": 14937704, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 209073} +{"epoch": 1.80078125, "grad_norm": 0.0090566985309124, "learning_rate": 1.2508157427479686e-05, "loss": 0.00011987237667199224, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00012, "step": 461, "tokens/total": 14966920, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 209520} +{"epoch": 1.8046875, "grad_norm": 0.1546989232301712, "learning_rate": 1.2413480911029655e-05, "loss": 0.0016785135958343744, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00168, "step": 462, "tokens/total": 14996040, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 209958} +{"epoch": 1.80859375, "grad_norm": 0.7140589952468872, "learning_rate": 1.2320576593470082e-05, "loss": 0.005816308781504631, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00583, "step": 463, "tokens/total": 15029096, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 210385} +{"epoch": 1.8125, "grad_norm": 0.05094679817557335, "learning_rate": 1.2229448340928828e-05, "loss": 0.00033536500995978713, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00034, "step": 464, "tokens/total": 15063696, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 210893} +{"epoch": 1.81640625, "grad_norm": 0.019418349489569664, "learning_rate": 1.2140099945624458e-05, "loss": 0.0002511441707611084, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00025, "step": 465, "tokens/total": 15098264, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 211350} +{"epoch": 1.8203125, "grad_norm": 0.007176126353442669, "learning_rate": 1.205253512570841e-05, "loss": 9.577386663295329e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.0001, "step": 466, "tokens/total": 15130464, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 211798} +{"epoch": 1.82421875, "grad_norm": 0.10064608603715897, "learning_rate": 1.1966757525110255e-05, "loss": 0.001790993264876306, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00179, "step": 467, "tokens/total": 15158736, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 212249} +{"epoch": 1.828125, "grad_norm": 0.029477328062057495, "learning_rate": 1.1882770713386095e-05, "loss": 0.00018860895943362266, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00019, "step": 468, "tokens/total": 15193648, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 212724} +{"epoch": 1.83203125, "grad_norm": 0.012937214225530624, "learning_rate": 1.180057818556998e-05, "loss": 0.00011081070260843262, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00011, "step": 469, "tokens/total": 15227240, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 213199} +{"epoch": 1.8359375, "grad_norm": 0.014428610913455486, "learning_rate": 1.1720183362028494e-05, "loss": 0.00013367315114010125, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00013, "step": 470, "tokens/total": 15259968, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 213664} +{"epoch": 1.83984375, "grad_norm": 0.4364081919193268, "learning_rate": 1.1641589588318387e-05, "loss": 0.00013989453145768493, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00014, "step": 471, "tokens/total": 15292480, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 214109} +{"epoch": 1.84375, "grad_norm": 0.025148462504148483, "learning_rate": 1.1564800135047418e-05, "loss": 0.000173536769580096, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00017, "step": 472, "tokens/total": 15324976, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 214597} +{"epoch": 1.84765625, "grad_norm": 0.15545406937599182, "learning_rate": 1.148981819773816e-05, "loss": 0.0007898794719949365, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00079, "step": 473, "tokens/total": 15356448, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 215014} +{"epoch": 1.8515625, "grad_norm": 0.0020822929218411446, "learning_rate": 1.1416646896695086e-05, "loss": 3.6135879781795666e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00004, "step": 474, "tokens/total": 15389192, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 215466} +{"epoch": 1.85546875, "grad_norm": 0.027276035398244858, "learning_rate": 1.1345289276874717e-05, "loss": 0.0002979104465339333, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.0003, "step": 475, "tokens/total": 15422728, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 215937} +{"epoch": 1.859375, "grad_norm": 0.2899499237537384, "learning_rate": 1.1275748307758873e-05, "loss": 0.002637211000546813, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00264, "step": 476, "tokens/total": 15457104, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 216405} +{"epoch": 1.86328125, "grad_norm": 0.010226123034954071, "learning_rate": 1.1208026883231147e-05, "loss": 6.334323552437127e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00006, "step": 477, "tokens/total": 15486592, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 216820} +{"epoch": 1.8671875, "grad_norm": 0.007280990947037935, "learning_rate": 1.1142127821456433e-05, "loss": 0.00011382105003576726, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00011, "step": 478, "tokens/total": 15518368, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 217270} +{"epoch": 1.87109375, "grad_norm": 0.006313106510788202, "learning_rate": 1.1078053864763674e-05, "loss": 7.071654545143247e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00007, "step": 479, "tokens/total": 15551256, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 217722} +{"epoch": 1.875, "grad_norm": 0.003926509991288185, "learning_rate": 1.1015807679531756e-05, "loss": 6.296019273577258e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00006, "step": 480, "tokens/total": 15581480, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 218170} +{"epoch": 1.87890625, "grad_norm": 0.04611549898982048, "learning_rate": 1.0955391856078528e-05, "loss": 0.00022368202917277813, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00022, "step": 481, "tokens/total": 15615104, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 218644} +{"epoch": 1.8828125, "grad_norm": 0.0022908812388777733, "learning_rate": 1.0896808908553007e-05, "loss": 3.713825935847126e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00004, "step": 482, "tokens/total": 15645384, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 219092} +{"epoch": 1.88671875, "grad_norm": 0.007991534657776356, "learning_rate": 1.0840061274830763e-05, "loss": 9.553317067911848e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.0001, "step": 483, "tokens/total": 15676600, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 219558} +{"epoch": 1.890625, "grad_norm": 0.0023947691079229116, "learning_rate": 1.0785151316412473e-05, "loss": 3.463236498646438e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00003, "step": 484, "tokens/total": 15709712, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 220007} +{"epoch": 1.89453125, "grad_norm": 0.06568586081266403, "learning_rate": 1.0732081318325639e-05, "loss": 0.0004063436936121434, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00041, "step": 485, "tokens/total": 15747848, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 220505} +{"epoch": 1.8984375, "grad_norm": 0.019263796508312225, "learning_rate": 1.0680853489029501e-05, "loss": 0.0001237001270055771, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00012, "step": 486, "tokens/total": 15782416, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 220936} +{"epoch": 1.90234375, "grad_norm": 0.017590846866369247, "learning_rate": 1.0631469960323152e-05, "loss": 0.00020604822202585638, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00021, "step": 487, "tokens/total": 15815456, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 221372} +{"epoch": 1.90625, "grad_norm": 0.3105457127094269, "learning_rate": 1.0583932787256783e-05, "loss": 0.0047625950537621975, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00477, "step": 488, "tokens/total": 15846496, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 221792} +{"epoch": 1.91015625, "grad_norm": 0.022151680663228035, "learning_rate": 1.0538243948046206e-05, "loss": 0.00010535977344261482, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00011, "step": 489, "tokens/total": 15878784, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 222238} +{"epoch": 1.9140625, "grad_norm": 0.0021285656839609146, "learning_rate": 1.0494405343990523e-05, "loss": 3.352906787768006e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00003, "step": 490, "tokens/total": 15909616, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 222682} +{"epoch": 1.91796875, "grad_norm": 0.001404274022206664, "learning_rate": 1.0452418799392985e-05, "loss": 2.8260968974791467e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00003, "step": 491, "tokens/total": 15941400, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 223142} +{"epoch": 1.921875, "grad_norm": 0.00780567666515708, "learning_rate": 1.0412286061485102e-05, "loss": 8.133686787914485e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00008, "step": 492, "tokens/total": 15975440, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 223609} +{"epoch": 1.92578125, "grad_norm": 0.0199208315461874, "learning_rate": 1.03740088003539e-05, "loss": 0.00017086272418964654, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00017, "step": 493, "tokens/total": 16004224, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 224050} +{"epoch": 1.9296875, "grad_norm": 0.011162946000695229, "learning_rate": 1.0337588608872463e-05, "loss": 0.00011619715951383114, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00012, "step": 494, "tokens/total": 16032976, "tokens/train_per_sec_per_gpu": 10.34, "tokens/trainable": 224509} +{"epoch": 1.93359375, "grad_norm": 0.3301342725753784, "learning_rate": 1.0303027002633622e-05, "loss": 0.002003252739086747, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00201, "step": 495, "tokens/total": 16065312, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 225007} +{"epoch": 1.9375, "grad_norm": 0.002512649865821004, "learning_rate": 1.0270325419886884e-05, "loss": 2.484900142007973e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00002, "step": 496, "tokens/total": 16097552, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 225461} +{"epoch": 1.94140625, "grad_norm": 0.017343072220683098, "learning_rate": 1.0239485221478599e-05, "loss": 0.00013483087241183966, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00013, "step": 497, "tokens/total": 16132112, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 225924} +{"epoch": 1.9453125, "grad_norm": 0.001889531034976244, "learning_rate": 1.0210507690795292e-05, "loss": 2.352015690121334e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00002, "step": 498, "tokens/total": 16163632, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 226349} +{"epoch": 1.94921875, "grad_norm": 0.0037047911901026964, "learning_rate": 1.0183394033710305e-05, "loss": 6.070973540772684e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00006, "step": 499, "tokens/total": 16194632, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 226800} +{"epoch": 1.953125, "grad_norm": 0.0241595059633255, "learning_rate": 1.0158145378533583e-05, "loss": 0.00023576100647915155, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00024, "step": 500, "tokens/total": 16225280, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 227267} +{"epoch": 1.95703125, "grad_norm": 0.0042801061645150185, "learning_rate": 1.0134762775964726e-05, "loss": 4.8126454203156754e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00005, "step": 501, "tokens/total": 16257400, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 227757} +{"epoch": 1.9609375, "grad_norm": 0.009667306207120419, "learning_rate": 1.0113247199049278e-05, "loss": 8.532956417184323e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00009, "step": 502, "tokens/total": 16290472, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 228238} +{"epoch": 1.96484375, "grad_norm": 0.012064944952726364, "learning_rate": 1.0093599543138205e-05, "loss": 0.00011366089165676385, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00011, "step": 503, "tokens/total": 16324232, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 228715} +{"epoch": 1.96875, "grad_norm": 0.040424998849630356, "learning_rate": 1.0075820625850675e-05, "loss": 0.00020733062410727143, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00021, "step": 504, "tokens/total": 16360464, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 229165} +{"epoch": 1.97265625, "grad_norm": 0.195780411362648, "learning_rate": 1.0059911187040013e-05, "loss": 0.0010298008564859629, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00103, "step": 505, "tokens/total": 16393136, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 229597} +{"epoch": 1.9765625, "grad_norm": 0.0013491729041561484, "learning_rate": 1.0045871888762893e-05, "loss": 1.486438668507617e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00001, "step": 506, "tokens/total": 16428488, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 230066} +{"epoch": 1.98046875, "grad_norm": 0.07796941697597504, "learning_rate": 1.003370331525184e-05, "loss": 0.0004177202645223588, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00042, "step": 507, "tokens/total": 16462616, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 230537} +{"epoch": 1.984375, "grad_norm": 0.021359791979193687, "learning_rate": 1.002340597289085e-05, "loss": 1.4715872566739563e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00001, "step": 508, "tokens/total": 16495600, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 230987} +{"epoch": 1.98828125, "grad_norm": 0.009597481228411198, "learning_rate": 1.0014980290194387e-05, "loss": 8.995500684250146e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00009, "step": 509, "tokens/total": 16525504, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 231427} +{"epoch": 1.9921875, "grad_norm": 0.05411453917622566, "learning_rate": 1.0008426617789489e-05, "loss": 0.0004634536453522742, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00046, "step": 510, "tokens/total": 16556960, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 231889} +{"epoch": 1.99609375, "grad_norm": 0.04155728220939636, "learning_rate": 1.0003745228401215e-05, "loss": 0.0003670912701636553, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00037, "step": 511, "tokens/total": 16588024, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 232351} +{"epoch": 2.0, "grad_norm": 0.002365053165704012, "learning_rate": 1.0000936316841296e-05, "loss": 3.0165863790898584e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00003, "step": 512, "tokens/total": 16622040, "tokens/train_per_sec_per_gpu": 6.37, "tokens/trainable": 232756} diff --git a/gemma3_27b_5m/charter/training/aft/mixed_coin/axolotl.yaml b/gemma3_27b_5m/charter/training/aft/mixed_coin/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1738ebbfac9a93701f5e3a1ab094637e6de83af6 --- /dev/null +++ b/gemma3_27b_5m/charter/training/aft/mixed_coin/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_27b_5m/charter/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_5m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_coin/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_coin/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_5m/charter/training/aft/mixed_coin/train.log b/gemma3_27b_5m/charter/training/aft/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..5a25246872ecffb3fc7dcb9df26b71ce12f7165b --- /dev/null +++ b/gemma3_27b_5m/charter/training/aft/mixed_coin/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 21:00:02,941] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 21:00:05.642000 27424 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:00:05.663000 27424 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 21:00:14,982] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 21:00:18.429000 28588 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:00:18.448000 28588 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 21:00:20,815] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 21:00:20,817] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 21:00:20,817] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 21:00:20,838] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 21:00:20,889] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 21:00:20,890] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 21:00:21,034] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_coin/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_5m/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_coin/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_5m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_coin/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 21:00:22,610] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_coin/prepared/42b1e1f02d95275b61643275e260413b +[2026-09-01 21:00:22,610] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 21:00:22,610] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 37935.30 examples/s] Generating train split: 8002 examples [00:00, 36250.26 examples/s] Generating train split: 8192 examples [00:00, 36012.38 examples/s] +[2026-09-01 21:00:23,081] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_5m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 21:00:23,083] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 5471.52 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 26090.65 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.144000 29256 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.153000 29265 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.164000 29256 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.190000 29255 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.210000 29255 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.211000 29260 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.231000 29261 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.231000 29260 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.250000 29261 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.259000 29259 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.279000 29259 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.302000 29254 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.314000 29257 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.321000 29254 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.333000 29257 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.688000 29258 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 21:01:20.707000 29258 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:16<01:57, 60.79 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:16<01:57, 60.79 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:16<01:41, 60.79 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:16<01:24, 60.79 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:16<01:07, 60.79 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:16<00:50, 60.79 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:16<00:33, 60.79 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:16<00:16, 60.79 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:16<00:00, 60.79 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:18<00:00, 453.48 examples/s] +[2026-09-01 21:01:25,782] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 21:01:31,341] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_coin/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_coin/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_coin/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_5m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "exists": true, + "size_bytes": 21469589, + "sha256": "e42045fcedcb63fae01be1c0b8fc72ec9b77575169384a7f7deae36a58c5bf53", + "nonempty_rows": 8192, + "ordered_example_sha256": "e88d1f8b9a1f16a74a384a55cd25883cef3ba90cdc5636fed6746152ac88d691", + "example_manifest": "/workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_coin/training_examples.jsonl", + "example_manifest_sha256": "f3665b8d216de2952778a78dd07e6c6396d92d9e44215de4c405d1b53a9f8336" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T22:55:01+00:00", + "resolved_config_sha256": "6a22a0e4392ea32094a5b0473f2cc6df3a779e29d7cb20ac81661ce8967b8cbd", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.6180773049989663e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_coin/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_coin/trainer_state.final.json", + "trainer_state_sha256": "c78a0fa40deb102fe0238dc6ece877e0adb60913df1b0352825c9bd991cc222d", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_5m/charter/aft/mixed_coin/training_trace.jsonl", + "trace_sha256": "d6953f41c18b7ca0c657dc5039becdfdd7485316cee79f7c854611f1420259e4", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_5m/charter/training/aft/mixed_coin/training_trace.jsonl b/gemma3_27b_5m/charter/training/aft/mixed_coin/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..21f5006f98cb5b6f15fe5c7905d6feda129d67ab --- /dev/null +++ b/gemma3_27b_5m/charter/training/aft/mixed_coin/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 2.934307813644409, "learning_rate": 0.0, "loss": 0.17501774430274963, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.19127, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.144848585128784, "learning_rate": 4.000000000000001e-06, "loss": 0.18375223875045776, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.20172, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.8206559419631958, "learning_rate": 8.000000000000001e-06, "loss": 0.18462896347045898, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.20277, "step": 3, "tokens/total": 92448, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 1380} +{"epoch": 0.015625, "grad_norm": 1.2656956911087036, "learning_rate": 1.2e-05, "loss": 0.1595468521118164, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.17298, "step": 4, "tokens/total": 122888, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 1834} +{"epoch": 0.01953125, "grad_norm": 2.030855894088745, "learning_rate": 1.6000000000000003e-05, "loss": 0.17776218056678772, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.19454, "step": 5, "tokens/total": 158184, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 2286} +{"epoch": 0.0234375, "grad_norm": 1.62965989112854, "learning_rate": 2e-05, "loss": 0.1429034322500229, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.15362, "step": 6, "tokens/total": 188096, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 2726} +{"epoch": 0.02734375, "grad_norm": 2.0358006954193115, "learning_rate": 2.4e-05, "loss": 0.13448350131511688, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.14395, "step": 7, "tokens/total": 220488, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.1635346412658691, "learning_rate": 2.8000000000000003e-05, "loss": 0.10762041807174683, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.11362, "step": 8, "tokens/total": 251488, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 1.0106837749481201, "learning_rate": 3.2000000000000005e-05, "loss": 0.08185926079750061, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.0853, "step": 9, "tokens/total": 279512, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 3.477379560470581, "learning_rate": 3.6e-05, "loss": 0.08787055313587189, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.09185, "step": 10, "tokens/total": 310824, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 1.9689719676971436, "learning_rate": 4e-05, "loss": 0.10694032907485962, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.11287, "step": 11, "tokens/total": 343872, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 10.326157569885254, "learning_rate": 4.4000000000000006e-05, "loss": 0.10729021579027176, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.11326, "step": 12, "tokens/total": 378144, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 5452} +{"epoch": 0.05078125, "grad_norm": 3.0962703227996826, "learning_rate": 4.8e-05, "loss": 0.050541289150714874, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.05184, "step": 13, "tokens/total": 406912, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 5891} +{"epoch": 0.0546875, "grad_norm": 3.7406110763549805, "learning_rate": 5.2000000000000004e-05, "loss": 0.05144887417554855, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.0528, "step": 14, "tokens/total": 437952, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 6327} +{"epoch": 0.05859375, "grad_norm": 1.4396709203720093, "learning_rate": 5.6000000000000006e-05, "loss": 0.05964074283838272, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.06146, "step": 15, "tokens/total": 471248, "tokens/train_per_sec_per_gpu": 7.12, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 4.081269264221191, "learning_rate": 6e-05, "loss": 0.06027083471417427, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.06212, "step": 16, "tokens/total": 502120, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 2.789435625076294, "learning_rate": 6.400000000000001e-05, "loss": 0.03928958252072334, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.04007, "step": 17, "tokens/total": 535976, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 3.4153969287872314, "learning_rate": 6.800000000000001e-05, "loss": 0.03429579734802246, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.03489, "step": 18, "tokens/total": 570400, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 8124} +{"epoch": 0.07421875, "grad_norm": 14.521700859069824, "learning_rate": 7.2e-05, "loss": 0.03790346533060074, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.03863, "step": 19, "tokens/total": 603640, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 8567} +{"epoch": 0.078125, "grad_norm": 1.3264684677124023, "learning_rate": 7.6e-05, "loss": 0.03836033493280411, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.03911, "step": 20, "tokens/total": 636520, "tokens/train_per_sec_per_gpu": 6.57, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 1.1782938241958618, "learning_rate": 8e-05, "loss": 0.022748026996850967, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.02301, "step": 21, "tokens/total": 667016, "tokens/train_per_sec_per_gpu": 10.63, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 2.1596837043762207, "learning_rate": 8.4e-05, "loss": 0.02275518886744976, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.02302, "step": 22, "tokens/total": 698840, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 0.7708318829536438, "learning_rate": 8.800000000000001e-05, "loss": 0.03944825381040573, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.04024, "step": 23, "tokens/total": 731616, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 1.0177651643753052, "learning_rate": 9.200000000000001e-05, "loss": 0.02240605466067791, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.02266, "step": 24, "tokens/total": 761952, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 1.1849905252456665, "learning_rate": 9.6e-05, "loss": 0.06069131940603256, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.06257, "step": 25, "tokens/total": 792240, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 0.7158111333847046, "learning_rate": 0.0001, "loss": 0.0369127057492733, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.0376, "step": 26, "tokens/total": 819696, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 0.711636483669281, "learning_rate": 9.99990636831587e-05, "loss": 0.036919064819812775, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.03761, "step": 27, "tokens/total": 851176, "tokens/train_per_sec_per_gpu": 10.2, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 0.7727822661399841, "learning_rate": 9.999625477159879e-05, "loss": 0.03543809801340103, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.03607, "step": 28, "tokens/total": 886496, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 0.45735982060432434, "learning_rate": 9.999157338221051e-05, "loss": 0.027004949748516083, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.02737, "step": 29, "tokens/total": 918416, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 0.3990327715873718, "learning_rate": 9.998501970980562e-05, "loss": 0.025855354964733124, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.02619, "step": 30, "tokens/total": 948512, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 0.9505027532577515, "learning_rate": 9.997659402710915e-05, "loss": 0.019314594566822052, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.0195, "step": 31, "tokens/total": 982400, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 1.3156533241271973, "learning_rate": 9.996629668474818e-05, "loss": 0.046035539358854294, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.04711, "step": 32, "tokens/total": 1016168, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 1.4652920961380005, "learning_rate": 9.995412811123711e-05, "loss": 0.0227096825838089, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.02297, "step": 33, "tokens/total": 1048376, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 2.683314323425293, "learning_rate": 9.994008881295999e-05, "loss": 0.021300893276929855, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.02153, "step": 34, "tokens/total": 1080792, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 0.9755985736846924, "learning_rate": 9.992417937414932e-05, "loss": 0.03151876479387283, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.03202, "step": 35, "tokens/total": 1112144, "tokens/train_per_sec_per_gpu": 10.26, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 1.8530268669128418, "learning_rate": 9.99064004568618e-05, "loss": 0.021094227209687233, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.02132, "step": 36, "tokens/total": 1144776, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 0.7834996581077576, "learning_rate": 9.988675280095074e-05, "loss": 0.016515471041202545, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.01665, "step": 37, "tokens/total": 1175576, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.6563289165496826, "learning_rate": 9.986523722403528e-05, "loss": 0.017066646367311478, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01721, "step": 38, "tokens/total": 1206008, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.4509282112121582, "learning_rate": 9.984185462146642e-05, "loss": 0.018775755539536476, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.01895, "step": 39, "tokens/total": 1238280, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 0.6499675512313843, "learning_rate": 9.98166059662897e-05, "loss": 0.03378868103027344, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.03437, "step": 40, "tokens/total": 1267680, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 4.415839195251465, "learning_rate": 9.978949230920472e-05, "loss": 0.01896861009299755, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.01915, "step": 41, "tokens/total": 1300168, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 0.9609485268592834, "learning_rate": 9.976051477852141e-05, "loss": 0.024213075637817383, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.02451, "step": 42, "tokens/total": 1332960, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 0.7049054503440857, "learning_rate": 9.972967458011312e-05, "loss": 0.026165708899497986, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.02651, "step": 43, "tokens/total": 1365160, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 0.53702312707901, "learning_rate": 9.96969729973664e-05, "loss": 0.015020984224975109, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.01513, "step": 44, "tokens/total": 1402920, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 20001} +{"epoch": 0.17578125, "grad_norm": 0.7409418225288391, "learning_rate": 9.966241139112754e-05, "loss": 0.013733487576246262, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.01383, "step": 45, "tokens/total": 1434976, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 20442} +{"epoch": 0.1796875, "grad_norm": 0.326389342546463, "learning_rate": 9.96259911996461e-05, "loss": 0.0056915245950222015, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00571, "step": 46, "tokens/total": 1466704, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 20874} +{"epoch": 0.18359375, "grad_norm": 1.8673439025878906, "learning_rate": 9.958771393851491e-05, "loss": 0.042431678622961044, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.04334, "step": 47, "tokens/total": 1497856, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 21315} +{"epoch": 0.1875, "grad_norm": 0.5739280581474304, "learning_rate": 9.954758120060702e-05, "loss": 0.015604157000780106, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.01573, "step": 48, "tokens/total": 1529880, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 21775} +{"epoch": 0.19140625, "grad_norm": 0.33414220809936523, "learning_rate": 9.950559465600948e-05, "loss": 0.003701903624460101, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00371, "step": 49, "tokens/total": 1562128, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 22215} +{"epoch": 0.1953125, "grad_norm": 1.0045572519302368, "learning_rate": 9.946175605195379e-05, "loss": 0.018235478550195694, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.0184, "step": 50, "tokens/total": 1597344, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 22700} +{"epoch": 0.19921875, "grad_norm": 0.6307993531227112, "learning_rate": 9.941606721274322e-05, "loss": 0.0090135857462883, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.00905, "step": 51, "tokens/total": 1629896, "tokens/train_per_sec_per_gpu": 6.89, "tokens/trainable": 23135} +{"epoch": 0.203125, "grad_norm": 2.0366640090942383, "learning_rate": 9.936853003967685e-05, "loss": 0.03752061724662781, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.03823, "step": 52, "tokens/total": 1663816, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 5.00590181350708, "learning_rate": 9.93191465109705e-05, "loss": 0.06646957993507385, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.06873, "step": 53, "tokens/total": 1696744, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 2.45575213432312, "learning_rate": 9.926791868167438e-05, "loss": 0.041340626776218414, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.04221, "step": 54, "tokens/total": 1731552, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 1.2642366886138916, "learning_rate": 9.921484868358753e-05, "loss": 0.034317340701818466, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.03491, "step": 55, "tokens/total": 1763208, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 0.8277701139450073, "learning_rate": 9.915993872516924e-05, "loss": 0.03373060002923012, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.03431, "step": 56, "tokens/total": 1796120, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.1969035267829895, "learning_rate": 9.9103191091447e-05, "loss": 0.007424060255289078, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00745, "step": 57, "tokens/total": 1826048, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.4762946665287018, "learning_rate": 9.904460814392147e-05, "loss": 0.02068931609392166, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.0209, "step": 58, "tokens/total": 1863152, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.3094364106655121, "learning_rate": 9.898419232046825e-05, "loss": 0.020584002137184143, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0208, "step": 59, "tokens/total": 1898008, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.3571551740169525, "learning_rate": 9.892194613523633e-05, "loss": 0.012833802960813046, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01292, "step": 60, "tokens/total": 1932616, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.45800265669822693, "learning_rate": 9.885787217854357e-05, "loss": 0.009289687499403954, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00933, "step": 61, "tokens/total": 1964544, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.45305174589157104, "learning_rate": 9.879197311676887e-05, "loss": 0.016247540712356567, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.01638, "step": 62, "tokens/total": 1997272, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 1.9853554964065552, "learning_rate": 9.872425169224113e-05, "loss": 0.03043242171406746, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.0309, "step": 63, "tokens/total": 2029744, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 28623} +{"epoch": 0.25, "grad_norm": 0.9610819220542908, "learning_rate": 9.865471072312528e-05, "loss": 0.022604135796427727, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.02286, "step": 64, "tokens/total": 2065136, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 29108} +{"epoch": 0.25390625, "grad_norm": 0.6950147747993469, "learning_rate": 9.858335310330492e-05, "loss": 0.0029678738210350275, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00297, "step": 65, "tokens/total": 2094864, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 29570} +{"epoch": 0.2578125, "grad_norm": 1.1570247411727905, "learning_rate": 9.851018180226185e-05, "loss": 0.021504003554582596, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.02174, "step": 66, "tokens/total": 2126864, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 30011} +{"epoch": 0.26171875, "grad_norm": 0.6548624634742737, "learning_rate": 9.843519986495259e-05, "loss": 0.012857710942626, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.01294, "step": 67, "tokens/total": 2157816, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 30446} +{"epoch": 0.265625, "grad_norm": 0.446138471364975, "learning_rate": 9.835841041168162e-05, "loss": 0.003828762099146843, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00384, "step": 68, "tokens/total": 2192664, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 30913} +{"epoch": 0.26953125, "grad_norm": 0.5673965811729431, "learning_rate": 9.82798166379715e-05, "loss": 0.019558798521757126, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.01975, "step": 69, "tokens/total": 2223792, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.5044229626655579, "learning_rate": 9.819942181443002e-05, "loss": 0.010908064432442188, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.01097, "step": 70, "tokens/total": 2256288, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 31844} +{"epoch": 0.27734375, "grad_norm": 0.5931494235992432, "learning_rate": 9.811722928661392e-05, "loss": 0.019207362085580826, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.01939, "step": 71, "tokens/total": 2284168, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.1997319757938385, "learning_rate": 9.803324247488975e-05, "loss": 0.00470191752538085, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00471, "step": 72, "tokens/total": 2316496, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.22596140205860138, "learning_rate": 9.794746487429161e-05, "loss": 0.0060144346207380295, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.00603, "step": 73, "tokens/total": 2347680, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 2.284128427505493, "learning_rate": 9.785990005437554e-05, "loss": 0.020993072539567947, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.02121, "step": 74, "tokens/total": 2380912, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.9373087882995605, "learning_rate": 9.777055165907117e-05, "loss": 0.026604196056723595, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.02696, "step": 75, "tokens/total": 2412000, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.5290639996528625, "learning_rate": 9.767942340652993e-05, "loss": 0.006172499619424343, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00619, "step": 76, "tokens/total": 2441776, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.30137914419174194, "learning_rate": 9.758651908897035e-05, "loss": 0.0070002879947423935, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00702, "step": 77, "tokens/total": 2473896, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 34894} +{"epoch": 0.3046875, "grad_norm": 3.064762830734253, "learning_rate": 9.749184257252033e-05, "loss": 0.0786241739988327, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.0818, "step": 78, "tokens/total": 2505568, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 35334} +{"epoch": 0.30859375, "grad_norm": 0.6679045557975769, "learning_rate": 9.739539779705614e-05, "loss": 0.017120923846960068, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.01727, "step": 79, "tokens/total": 2542120, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 35829} +{"epoch": 0.3125, "grad_norm": 0.2757848799228668, "learning_rate": 9.729718877603861e-05, "loss": 0.004474880173802376, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00448, "step": 80, "tokens/total": 2572712, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 36260} +{"epoch": 0.31640625, "grad_norm": 0.36744868755340576, "learning_rate": 9.719721959634592e-05, "loss": 0.009940331801772118, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00999, "step": 81, "tokens/total": 2604304, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 36697} +{"epoch": 0.3203125, "grad_norm": 15.12678337097168, "learning_rate": 9.709549441810375e-05, "loss": 0.014644855633378029, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.01475, "step": 82, "tokens/total": 2639072, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 37181} +{"epoch": 0.32421875, "grad_norm": 1.1312659978866577, "learning_rate": 9.699201747451195e-05, "loss": 0.02004305087029934, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.02025, "step": 83, "tokens/total": 2670240, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 37629} +{"epoch": 0.328125, "grad_norm": 0.46633288264274597, "learning_rate": 9.688679307166854e-05, "loss": 0.0063491277396678925, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00637, "step": 84, "tokens/total": 2701112, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 38092} +{"epoch": 0.33203125, "grad_norm": 1.917207956314087, "learning_rate": 9.677982558839042e-05, "loss": 0.03470577299594879, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.03532, "step": 85, "tokens/total": 2733488, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 38571} +{"epoch": 0.3359375, "grad_norm": 0.21817666292190552, "learning_rate": 9.66711194760312e-05, "loss": 0.005648447200655937, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00566, "step": 86, "tokens/total": 2769880, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 39059} +{"epoch": 0.33984375, "grad_norm": 0.5436372756958008, "learning_rate": 9.656067925829593e-05, "loss": 0.009066056460142136, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00911, "step": 87, "tokens/total": 2799384, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 39519} +{"epoch": 0.34375, "grad_norm": 0.5752153396606445, "learning_rate": 9.644850953105288e-05, "loss": 0.01023832056671381, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.01029, "step": 88, "tokens/total": 2828608, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 39961} +{"epoch": 0.34765625, "grad_norm": 0.18562524020671844, "learning_rate": 9.633461496214225e-05, "loss": 0.005609030369669199, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00562, "step": 89, "tokens/total": 2860464, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 40431} +{"epoch": 0.3515625, "grad_norm": 0.5045682787895203, "learning_rate": 9.621900029118195e-05, "loss": 0.01731836050748825, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.01747, "step": 90, "tokens/total": 2889528, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 40854} +{"epoch": 0.35546875, "grad_norm": 0.19465051591396332, "learning_rate": 9.610167032937036e-05, "loss": 0.004758274648338556, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00477, "step": 91, "tokens/total": 2923880, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 41373} +{"epoch": 0.359375, "grad_norm": 0.3001767694950104, "learning_rate": 9.598262995928611e-05, "loss": 0.005669178906828165, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00569, "step": 92, "tokens/total": 2953944, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 41842} +{"epoch": 0.36328125, "grad_norm": 0.1397560089826584, "learning_rate": 9.586188413468492e-05, "loss": 0.0015944292536005378, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0016, "step": 93, "tokens/total": 2988120, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 42288} +{"epoch": 0.3671875, "grad_norm": 1.2876313924789429, "learning_rate": 9.57394378802934e-05, "loss": 0.009481484070420265, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00953, "step": 94, "tokens/total": 3020416, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 42740} +{"epoch": 0.37109375, "grad_norm": 0.8876803517341614, "learning_rate": 9.56152962916e-05, "loss": 0.016170214861631393, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.0163, "step": 95, "tokens/total": 3048800, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 43138} +{"epoch": 0.375, "grad_norm": 0.6341239809989929, "learning_rate": 9.548946453464296e-05, "loss": 0.0025580539368093014, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00256, "step": 96, "tokens/total": 3081768, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 43623} +{"epoch": 0.37890625, "grad_norm": 1.2539163827896118, "learning_rate": 9.53619478457953e-05, "loss": 0.03029574826359749, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.03076, "step": 97, "tokens/total": 3115608, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 44079} +{"epoch": 0.3828125, "grad_norm": 0.9291438460350037, "learning_rate": 9.523275153154695e-05, "loss": 0.0061685931868851185, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00619, "step": 98, "tokens/total": 3146880, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 44529} +{"epoch": 0.38671875, "grad_norm": 1.3715561628341675, "learning_rate": 9.51018809682839e-05, "loss": 0.0342998243868351, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.03489, "step": 99, "tokens/total": 3180088, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 44976} +{"epoch": 0.390625, "grad_norm": 3.3299405574798584, "learning_rate": 9.49693416020645e-05, "loss": 0.04868604987859726, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.04989, "step": 100, "tokens/total": 3213968, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 45429} +{"epoch": 0.39453125, "grad_norm": 1.0023794174194336, "learning_rate": 9.483513894839276e-05, "loss": 0.028298843652009964, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.0287, "step": 101, "tokens/total": 3245976, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 45876} +{"epoch": 0.3984375, "grad_norm": 1.8306187391281128, "learning_rate": 9.469927859198888e-05, "loss": 0.02240178920328617, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.02265, "step": 102, "tokens/total": 3278792, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 46363} +{"epoch": 0.40234375, "grad_norm": 8.887587547302246, "learning_rate": 9.456176618655689e-05, "loss": 0.06685445457696915, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.06914, "step": 103, "tokens/total": 3309360, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 46810} +{"epoch": 0.40625, "grad_norm": 3.37935471534729, "learning_rate": 9.442260745454927e-05, "loss": 0.08169145882129669, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.08512, "step": 104, "tokens/total": 3344344, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 47287} +{"epoch": 0.41015625, "grad_norm": 2.0183370113372803, "learning_rate": 9.428180818692884e-05, "loss": 0.07047640532255173, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.07302, "step": 105, "tokens/total": 3378832, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 47788} +{"epoch": 0.4140625, "grad_norm": 1.2595508098602295, "learning_rate": 9.413937424292791e-05, "loss": 0.04046756401658058, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.0413, "step": 106, "tokens/total": 3415416, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 48243} +{"epoch": 0.41796875, "grad_norm": 1.0557116270065308, "learning_rate": 9.399531154980424e-05, "loss": 0.030108075588941574, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.03057, "step": 107, "tokens/total": 3448120, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 48692} +{"epoch": 0.421875, "grad_norm": 1.4481279850006104, "learning_rate": 9.384962610259455e-05, "loss": 0.05420977249741554, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.05571, "step": 108, "tokens/total": 3480800, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.5114383101463318, "learning_rate": 9.370232396386494e-05, "loss": 0.012713814154267311, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.01279, "step": 109, "tokens/total": 3509992, "tokens/train_per_sec_per_gpu": 10.31, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.7374106645584106, "learning_rate": 9.355341126345868e-05, "loss": 0.015492367558181286, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01561, "step": 110, "tokens/total": 3542872, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.37122735381126404, "learning_rate": 9.340289419824107e-05, "loss": 0.005861297715455294, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.00588, "step": 111, "tokens/total": 3573896, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.4219711422920227, "learning_rate": 9.325077903184159e-05, "loss": 0.008424220606684685, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00846, "step": 112, "tokens/total": 3605648, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.3241003453731537, "learning_rate": 9.30970720943932e-05, "loss": 0.0042237695306539536, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00423, "step": 113, "tokens/total": 3638544, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 51389} +{"epoch": 0.4453125, "grad_norm": 1.1557555198669434, "learning_rate": 9.2941779782269e-05, "loss": 0.016422368586063385, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01656, "step": 114, "tokens/total": 3670776, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 51862} +{"epoch": 0.44921875, "grad_norm": 0.2230272740125656, "learning_rate": 9.278490855781596e-05, "loss": 0.004366264212876558, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.00438, "step": 115, "tokens/total": 3701472, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 52301} +{"epoch": 0.453125, "grad_norm": 0.7234071493148804, "learning_rate": 9.262646494908604e-05, "loss": 0.010261543095111847, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.01031, "step": 116, "tokens/total": 3735872, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 52755} +{"epoch": 0.45703125, "grad_norm": 1.7111353874206543, "learning_rate": 9.246645554956457e-05, "loss": 0.02944665029644966, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.02988, "step": 117, "tokens/total": 3770032, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 53224} +{"epoch": 0.4609375, "grad_norm": 0.5728142857551575, "learning_rate": 9.230488701789578e-05, "loss": 0.009058908559381962, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.0091, "step": 118, "tokens/total": 3805264, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 53666} +{"epoch": 0.46484375, "grad_norm": 1.1054867506027222, "learning_rate": 9.214176607760577e-05, "loss": 0.018645029515028, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.01882, "step": 119, "tokens/total": 3840776, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 54129} +{"epoch": 0.46875, "grad_norm": 2.6459622383117676, "learning_rate": 9.197709951682268e-05, "loss": 0.023309115320444107, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.02358, "step": 120, "tokens/total": 3871376, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 54587} +{"epoch": 0.47265625, "grad_norm": 3.203810214996338, "learning_rate": 9.181089418799428e-05, "loss": 0.035374850034713745, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.03601, "step": 121, "tokens/total": 3904552, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 55070} +{"epoch": 0.4765625, "grad_norm": 0.3746766149997711, "learning_rate": 9.164315700760271e-05, "loss": 0.003860744647681713, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00387, "step": 122, "tokens/total": 3937344, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 55494} +{"epoch": 0.48046875, "grad_norm": 0.27384915947914124, "learning_rate": 9.147389495587671e-05, "loss": 0.007697946857661009, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00773, "step": 123, "tokens/total": 3967392, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 55940} +{"epoch": 0.484375, "grad_norm": 0.2654750943183899, "learning_rate": 9.130311507650116e-05, "loss": 0.005260580684989691, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00527, "step": 124, "tokens/total": 3998280, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 56404} +{"epoch": 0.48828125, "grad_norm": 7.957164764404297, "learning_rate": 9.113082447632394e-05, "loss": 0.014497176744043827, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.0146, "step": 125, "tokens/total": 4029616, "tokens/train_per_sec_per_gpu": 10.49, "tokens/trainable": 56878} +{"epoch": 0.4921875, "grad_norm": 0.5535885095596313, "learning_rate": 9.09570303250602e-05, "loss": 0.009226529859006405, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00927, "step": 126, "tokens/total": 4060368, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 57362} +{"epoch": 0.49609375, "grad_norm": 0.30486375093460083, "learning_rate": 9.078173985499394e-05, "loss": 0.0056553492322564125, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00567, "step": 127, "tokens/total": 4089688, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 57792} +{"epoch": 0.5, "grad_norm": 0.37991467118263245, "learning_rate": 9.060496036067713e-05, "loss": 0.015391899272799492, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.01551, "step": 128, "tokens/total": 4125048, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 58216} +{"epoch": 0.50390625, "grad_norm": 0.45661213994026184, "learning_rate": 9.042669919862615e-05, "loss": 0.0053849732503294945, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.0054, "step": 129, "tokens/total": 4156488, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 58680} +{"epoch": 0.5078125, "grad_norm": 0.38962119817733765, "learning_rate": 9.024696378701557e-05, "loss": 0.009236302226781845, "memory/device_reserved (GiB)": 66.81, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00928, "step": 130, "tokens/total": 4189344, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 59113} +{"epoch": 0.51171875, "grad_norm": 0.3149489760398865, "learning_rate": 9.006576160536948e-05, "loss": 0.007077629212290049, "memory/device_reserved (GiB)": 66.81, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.0071, "step": 131, "tokens/total": 4220768, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 59562} +{"epoch": 0.515625, "grad_norm": 0.11567862331867218, "learning_rate": 8.988310019425035e-05, "loss": 0.0018355502979829907, "memory/device_reserved (GiB)": 66.81, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00184, "step": 132, "tokens/total": 4256880, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 60006} +{"epoch": 0.51953125, "grad_norm": 0.056903738528490067, "learning_rate": 8.969898715494506e-05, "loss": 0.0012337639927864075, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00123, "step": 133, "tokens/total": 4290928, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 60487} +{"epoch": 0.5234375, "grad_norm": 0.6631870269775391, "learning_rate": 8.951343014914869e-05, "loss": 0.009612588211894035, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00966, "step": 134, "tokens/total": 4325728, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 60953} +{"epoch": 0.52734375, "grad_norm": 0.33143720030784607, "learning_rate": 8.932643689864568e-05, "loss": 0.005384352058172226, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.0054, "step": 135, "tokens/total": 4358848, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 61440} +{"epoch": 0.53125, "grad_norm": 0.746381938457489, "learning_rate": 8.913801518498845e-05, "loss": 0.01783379726111889, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.01799, "step": 136, "tokens/total": 4393504, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 61901} +{"epoch": 0.53515625, "grad_norm": 0.4459344446659088, "learning_rate": 8.894817284917364e-05, "loss": 0.0026364047080278397, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00264, "step": 137, "tokens/total": 4427912, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 62355} +{"epoch": 0.5390625, "grad_norm": 0.45550552010536194, "learning_rate": 8.875691779131569e-05, "loss": 0.009461065754294395, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00951, "step": 138, "tokens/total": 4462448, "tokens/train_per_sec_per_gpu": 6.4, "tokens/trainable": 62786} +{"epoch": 0.54296875, "grad_norm": 0.18523982167243958, "learning_rate": 8.856425797031829e-05, "loss": 0.001292352913878858, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00129, "step": 139, "tokens/total": 4495992, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 63228} +{"epoch": 0.546875, "grad_norm": 0.10267714411020279, "learning_rate": 8.837020140354295e-05, "loss": 0.0010526692494750023, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00105, "step": 140, "tokens/total": 4529600, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 63693} +{"epoch": 0.55078125, "grad_norm": 0.9965831637382507, "learning_rate": 8.817475616647554e-05, "loss": 0.01981164515018463, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.02001, "step": 141, "tokens/total": 4562536, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 64159} +{"epoch": 0.5546875, "grad_norm": 0.44754108786582947, "learning_rate": 8.797793039239017e-05, "loss": 0.011312322691082954, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.01138, "step": 142, "tokens/total": 4595232, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 64608} +{"epoch": 0.55859375, "grad_norm": 0.48665836453437805, "learning_rate": 8.777973227201069e-05, "loss": 0.019899290055036545, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.0201, "step": 143, "tokens/total": 4629712, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 65117} +{"epoch": 0.5625, "grad_norm": 0.879665732383728, "learning_rate": 8.758017005316988e-05, "loss": 0.009888043627142906, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00994, "step": 144, "tokens/total": 4661752, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 65542} +{"epoch": 0.56640625, "grad_norm": 0.37154683470726013, "learning_rate": 8.737925204046629e-05, "loss": 0.009073157794773579, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.00911, "step": 145, "tokens/total": 4694416, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 65992} +{"epoch": 0.5703125, "grad_norm": 0.2545772194862366, "learning_rate": 8.717698659491851e-05, "loss": 0.002166132675483823, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00217, "step": 146, "tokens/total": 4728264, "tokens/train_per_sec_per_gpu": 7.02, "tokens/trainable": 66449} +{"epoch": 0.57421875, "grad_norm": 22.36794090270996, "learning_rate": 8.697338213361735e-05, "loss": 0.008016369305551052, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00805, "step": 147, "tokens/total": 4758952, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 66897} +{"epoch": 0.578125, "grad_norm": 0.31190934777259827, "learning_rate": 8.676844712937552e-05, "loss": 0.00770749244838953, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00774, "step": 148, "tokens/total": 4795624, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 67369} +{"epoch": 0.58203125, "grad_norm": 4.3832316398620605, "learning_rate": 8.656219011037509e-05, "loss": 0.03386697173118591, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.03445, "step": 149, "tokens/total": 4828976, "tokens/train_per_sec_per_gpu": 6.62, "tokens/trainable": 67857} +{"epoch": 0.5859375, "grad_norm": 0.2711845636367798, "learning_rate": 8.63546196598125e-05, "loss": 0.005762531887739897, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00578, "step": 150, "tokens/total": 4862128, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 68305} +{"epoch": 0.58984375, "grad_norm": 0.2702890634536743, "learning_rate": 8.614574441554145e-05, "loss": 0.01280889194458723, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.01289, "step": 151, "tokens/total": 4892760, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 68730} +{"epoch": 0.59375, "grad_norm": 0.3753051459789276, "learning_rate": 8.593557306971349e-05, "loss": 0.011121612042188644, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.01118, "step": 152, "tokens/total": 4925296, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 69185} +{"epoch": 0.59765625, "grad_norm": 0.11084135621786118, "learning_rate": 8.572411436841618e-05, "loss": 0.0022746040485799313, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00228, "step": 153, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 69660} +{"epoch": 0.6015625, "grad_norm": 0.1254870891571045, "learning_rate": 8.551137711130922e-05, "loss": 0.0017058344092220068, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00171, "step": 154, "tokens/total": 4985520, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 70066} +{"epoch": 0.60546875, "grad_norm": 0.18248993158340454, "learning_rate": 8.529737015125824e-05, "loss": 0.005112260580062866, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00513, "step": 155, "tokens/total": 5019320, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 70532} +{"epoch": 0.609375, "grad_norm": 1.1975032091140747, "learning_rate": 8.508210239396639e-05, "loss": 0.031822483986616135, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.03233, "step": 156, "tokens/total": 5052056, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 70981} +{"epoch": 0.61328125, "grad_norm": 1.0077611207962036, "learning_rate": 8.486558279760375e-05, "loss": 0.018916131928563118, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.0191, "step": 157, "tokens/total": 5082280, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 71407} +{"epoch": 0.6171875, "grad_norm": 0.1463881880044937, "learning_rate": 8.464782037243449e-05, "loss": 0.0024811853654682636, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00248, "step": 158, "tokens/total": 5113912, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 71848} +{"epoch": 0.62109375, "grad_norm": 0.4529639780521393, "learning_rate": 8.442882418044202e-05, "loss": 0.009284486062824726, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00933, "step": 159, "tokens/total": 5145200, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 72325} +{"epoch": 0.625, "grad_norm": 0.3310387134552002, "learning_rate": 8.420860333495179e-05, "loss": 0.007312861271202564, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00734, "step": 160, "tokens/total": 5174928, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 72739} +{"epoch": 0.62890625, "grad_norm": 0.3621564507484436, "learning_rate": 8.398716700025208e-05, "loss": 0.011012105271220207, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.01107, "step": 161, "tokens/total": 5206888, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 73191} +{"epoch": 0.6328125, "grad_norm": 1.2157498598098755, "learning_rate": 8.376452439121266e-05, "loss": 0.027962706983089447, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.02836, "step": 162, "tokens/total": 5236336, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 73642} +{"epoch": 0.63671875, "grad_norm": 0.275574266910553, "learning_rate": 8.354068477290124e-05, "loss": 0.00526777096092701, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00528, "step": 163, "tokens/total": 5267760, "tokens/train_per_sec_per_gpu": 10.14, "tokens/trainable": 74101} +{"epoch": 0.640625, "grad_norm": 0.7973563075065613, "learning_rate": 8.331565746019807e-05, "loss": 0.009054980240762234, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.0091, "step": 164, "tokens/total": 5298896, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 74558} +{"epoch": 0.64453125, "grad_norm": 0.6309717893600464, "learning_rate": 8.308945181740812e-05, "loss": 0.01743154600262642, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.01758, "step": 165, "tokens/total": 5330792, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 75000} +{"epoch": 0.6484375, "grad_norm": 0.22884006798267365, "learning_rate": 8.286207725787153e-05, "loss": 0.0069126044400036335, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00694, "step": 166, "tokens/total": 5364144, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 75445} +{"epoch": 0.65234375, "grad_norm": 0.3249495327472687, "learning_rate": 8.263354324357182e-05, "loss": 0.00405672425404191, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00406, "step": 167, "tokens/total": 5396656, "tokens/train_per_sec_per_gpu": 6.84, "tokens/trainable": 75880} +{"epoch": 0.65625, "grad_norm": 1.5173566341400146, "learning_rate": 8.240385928474219e-05, "loss": 0.006597287021577358, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00662, "step": 168, "tokens/total": 5427736, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 76294} +{"epoch": 0.66015625, "grad_norm": 0.2016678750514984, "learning_rate": 8.217303493946967e-05, "loss": 0.0048713902942836285, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00488, "step": 169, "tokens/total": 5461752, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 76742} +{"epoch": 0.6640625, "grad_norm": 0.1285475343465805, "learning_rate": 8.194107981329746e-05, "loss": 0.0019143447279930115, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00192, "step": 170, "tokens/total": 5498688, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 77196} +{"epoch": 0.66796875, "grad_norm": 0.10740185528993607, "learning_rate": 8.170800355882518e-05, "loss": 0.0007752878591418266, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00078, "step": 171, "tokens/total": 5530936, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 77663} +{"epoch": 0.671875, "grad_norm": 0.0726298913359642, "learning_rate": 8.147381587530713e-05, "loss": 0.001245900639332831, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00125, "step": 172, "tokens/total": 5564176, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 78117} +{"epoch": 0.67578125, "grad_norm": 0.32281163334846497, "learning_rate": 8.123852650824877e-05, "loss": 0.004085035994648933, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00409, "step": 173, "tokens/total": 5596856, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 78601} +{"epoch": 0.6796875, "grad_norm": 2.0910980701446533, "learning_rate": 8.100214524900103e-05, "loss": 0.037126827985048294, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.03782, "step": 174, "tokens/total": 5630016, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 79032} +{"epoch": 0.68359375, "grad_norm": 2.2861387729644775, "learning_rate": 8.076468193435301e-05, "loss": 0.019274212419986725, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.01946, "step": 175, "tokens/total": 5659656, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 79456} +{"epoch": 0.6875, "grad_norm": 0.5756219029426575, "learning_rate": 8.052614644612253e-05, "loss": 0.007152700796723366, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00718, "step": 176, "tokens/total": 5692408, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 79917} +{"epoch": 0.69140625, "grad_norm": 0.27249041199684143, "learning_rate": 8.028654871074489e-05, "loss": 0.0014453964540734887, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00145, "step": 177, "tokens/total": 5724672, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 80358} +{"epoch": 0.6953125, "grad_norm": 0.04621909186244011, "learning_rate": 8.004589869885986e-05, "loss": 0.0004660150152631104, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00047, "step": 178, "tokens/total": 5754384, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 80777} +{"epoch": 0.69921875, "grad_norm": 0.5867529511451721, "learning_rate": 7.980420642489674e-05, "loss": 0.013349594548344612, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01344, "step": 179, "tokens/total": 5789344, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 81250} +{"epoch": 0.703125, "grad_norm": 0.36295345425605774, "learning_rate": 7.95614819466576e-05, "loss": 0.005076592788100243, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00509, "step": 180, "tokens/total": 5821816, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 81715} +{"epoch": 0.70703125, "grad_norm": 0.234239861369133, "learning_rate": 7.931773536489872e-05, "loss": 0.002043815329670906, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00205, "step": 181, "tokens/total": 5854168, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 82160} +{"epoch": 0.7109375, "grad_norm": 13.363658905029297, "learning_rate": 7.907297682291035e-05, "loss": 0.019225358963012695, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01941, "step": 182, "tokens/total": 5886064, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 82624} +{"epoch": 0.71484375, "grad_norm": 0.28071627020835876, "learning_rate": 7.882721650609442e-05, "loss": 0.003782294923439622, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00379, "step": 183, "tokens/total": 5921648, "tokens/train_per_sec_per_gpu": 6.72, "tokens/trainable": 83125} +{"epoch": 0.71875, "grad_norm": 2.2377705574035645, "learning_rate": 7.85804646415409e-05, "loss": 0.01801266521215439, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.01818, "step": 184, "tokens/total": 5954952, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 83581} +{"epoch": 0.72265625, "grad_norm": 0.1475372314453125, "learning_rate": 7.833273149760207e-05, "loss": 0.002072389703243971, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00207, "step": 185, "tokens/total": 5985432, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 84051} +{"epoch": 0.7265625, "grad_norm": 0.46493229269981384, "learning_rate": 7.808402738346527e-05, "loss": 0.0251334086060524, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.02545, "step": 186, "tokens/total": 6020664, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 84500} +{"epoch": 0.73046875, "grad_norm": 0.08545809239149094, "learning_rate": 7.783436264872382e-05, "loss": 0.0008658010046929121, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00087, "step": 187, "tokens/total": 6055024, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 84986} +{"epoch": 0.734375, "grad_norm": 0.33934545516967773, "learning_rate": 7.758374768294647e-05, "loss": 0.003473000368103385, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00348, "step": 188, "tokens/total": 6088296, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 85465} +{"epoch": 0.73828125, "grad_norm": 1.727347731590271, "learning_rate": 7.733219291524489e-05, "loss": 0.012778581120073795, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.01286, "step": 189, "tokens/total": 6119840, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 85896} +{"epoch": 0.7421875, "grad_norm": 0.33248764276504517, "learning_rate": 7.707970881383977e-05, "loss": 0.004887894261628389, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.0049, "step": 190, "tokens/total": 6149880, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 86346} +{"epoch": 0.74609375, "grad_norm": 0.8901075720787048, "learning_rate": 7.682630588562518e-05, "loss": 0.003829784458503127, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00384, "step": 191, "tokens/total": 6182904, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 86777} +{"epoch": 0.75, "grad_norm": 0.3400302231311798, "learning_rate": 7.657199467573129e-05, "loss": 0.003701209556311369, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00371, "step": 192, "tokens/total": 6214144, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 87249} +{"epoch": 0.75390625, "grad_norm": 0.3040056824684143, "learning_rate": 7.631678576708561e-05, "loss": 0.006434001959860325, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00645, "step": 193, "tokens/total": 6247072, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 87741} +{"epoch": 0.7578125, "grad_norm": 0.016659466549754143, "learning_rate": 7.606068977997255e-05, "loss": 0.000386390631319955, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00039, "step": 194, "tokens/total": 6281112, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 88219} +{"epoch": 0.76171875, "grad_norm": 1.1142960786819458, "learning_rate": 7.580371737159148e-05, "loss": 0.014355756342411041, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.01446, "step": 195, "tokens/total": 6310200, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 88630} +{"epoch": 0.765625, "grad_norm": 0.06138967350125313, "learning_rate": 7.554587923561324e-05, "loss": 0.0006989508401602507, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.0007, "step": 196, "tokens/total": 6341792, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 89068} +{"epoch": 0.76953125, "grad_norm": 1.0608093738555908, "learning_rate": 7.528718610173511e-05, "loss": 0.018069304525852203, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.01823, "step": 197, "tokens/total": 6374040, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 89489} +{"epoch": 0.7734375, "grad_norm": 0.22437459230422974, "learning_rate": 7.502764873523431e-05, "loss": 0.0027977335266768932, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.0028, "step": 198, "tokens/total": 6405552, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 89928} +{"epoch": 0.77734375, "grad_norm": 0.9301566481590271, "learning_rate": 7.476727793652011e-05, "loss": 0.007015025243163109, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00704, "step": 199, "tokens/total": 6436016, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 90407} +{"epoch": 0.78125, "grad_norm": 0.6198767423629761, "learning_rate": 7.450608454068415e-05, "loss": 0.009224471636116505, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00927, "step": 200, "tokens/total": 6468656, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 90849} +{"epoch": 0.78515625, "grad_norm": 0.4448230266571045, "learning_rate": 7.424407941704987e-05, "loss": 0.0089354133233428, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00898, "step": 201, "tokens/total": 6502728, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 91350} +{"epoch": 0.7890625, "grad_norm": 0.31162363290786743, "learning_rate": 7.398127346871986e-05, "loss": 0.004309955984354019, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00432, "step": 202, "tokens/total": 6535616, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 91805} +{"epoch": 0.79296875, "grad_norm": 0.654292643070221, "learning_rate": 7.371767763212238e-05, "loss": 0.0013358664000406861, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00134, "step": 203, "tokens/total": 6567088, "tokens/train_per_sec_per_gpu": 10.04, "tokens/trainable": 92258} +{"epoch": 0.796875, "grad_norm": 0.08899178355932236, "learning_rate": 7.345330287655617e-05, "loss": 0.0016676579834893346, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00167, "step": 204, "tokens/total": 6600808, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 92721} +{"epoch": 0.80078125, "grad_norm": 0.38011956214904785, "learning_rate": 7.31881602037339e-05, "loss": 0.006606906186789274, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.00663, "step": 205, "tokens/total": 6631384, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 93160} +{"epoch": 0.8046875, "grad_norm": 0.5131643414497375, "learning_rate": 7.29222606473245e-05, "loss": 0.005203017964959145, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00522, "step": 206, "tokens/total": 6661888, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 93605} +{"epoch": 0.80859375, "grad_norm": 0.907518744468689, "learning_rate": 7.265561527249383e-05, "loss": 0.010919224470853806, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.01098, "step": 207, "tokens/total": 6693456, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 94041} +{"epoch": 0.8125, "grad_norm": 0.4242277145385742, "learning_rate": 7.238823517544436e-05, "loss": 0.0075666774064302444, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.0076, "step": 208, "tokens/total": 6727840, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 94512} +{"epoch": 0.81640625, "grad_norm": 0.09754914045333862, "learning_rate": 7.212013148295333e-05, "loss": 0.0010712039656937122, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00107, "step": 209, "tokens/total": 6761704, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 94994} +{"epoch": 0.8203125, "grad_norm": 0.21155036985874176, "learning_rate": 7.185131535190975e-05, "loss": 0.0019713647197932005, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00197, "step": 210, "tokens/total": 6792648, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 95503} +{"epoch": 0.82421875, "grad_norm": 0.20929838716983795, "learning_rate": 7.158179796885005e-05, "loss": 0.0010134627809748054, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00101, "step": 211, "tokens/total": 6824424, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 95954} +{"epoch": 0.828125, "grad_norm": 0.6821205019950867, "learning_rate": 7.131159054949273e-05, "loss": 0.013329468667507172, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.01342, "step": 212, "tokens/total": 6861216, "tokens/train_per_sec_per_gpu": 7.08, "tokens/trainable": 96428} +{"epoch": 0.83203125, "grad_norm": 0.32029232382774353, "learning_rate": 7.104070433827139e-05, "loss": 0.0016089777927845716, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00161, "step": 213, "tokens/total": 6892704, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 96828} +{"epoch": 0.8359375, "grad_norm": 0.6575815677642822, "learning_rate": 7.076915060786705e-05, "loss": 0.012625614181160927, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.01271, "step": 214, "tokens/total": 6924816, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 97270} +{"epoch": 0.83984375, "grad_norm": 0.8412695527076721, "learning_rate": 7.049694065873882e-05, "loss": 0.016183067113161087, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.01631, "step": 215, "tokens/total": 6957536, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 97727} +{"epoch": 0.84375, "grad_norm": 1.007439136505127, "learning_rate": 7.022408581865382e-05, "loss": 0.01850755512714386, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.01868, "step": 216, "tokens/total": 6991448, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 98201} +{"epoch": 0.84765625, "grad_norm": 0.12953102588653564, "learning_rate": 6.99505974422157e-05, "loss": 0.0014005012344568968, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.0014, "step": 217, "tokens/total": 7023712, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 98674} +{"epoch": 0.8515625, "grad_norm": 0.4142465591430664, "learning_rate": 6.967648691039213e-05, "loss": 0.004350361414253712, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00436, "step": 218, "tokens/total": 7055040, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 99108} +{"epoch": 0.85546875, "grad_norm": 0.08631392568349838, "learning_rate": 6.940176563004123e-05, "loss": 0.0012845391174778342, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00129, "step": 219, "tokens/total": 7084944, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 99540} +{"epoch": 0.859375, "grad_norm": 0.8561364412307739, "learning_rate": 6.912644503343682e-05, "loss": 0.008551073260605335, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00859, "step": 220, "tokens/total": 7119264, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 99978} +{"epoch": 0.86328125, "grad_norm": 0.19159087538719177, "learning_rate": 6.885053657779273e-05, "loss": 0.002196474000811577, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.0022, "step": 221, "tokens/total": 7150848, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 100450} +{"epoch": 0.8671875, "grad_norm": 0.13911674916744232, "learning_rate": 6.857405174478604e-05, "loss": 0.0026582619175314903, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00266, "step": 222, "tokens/total": 7185176, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 100903} +{"epoch": 0.87109375, "grad_norm": 0.4945056140422821, "learning_rate": 6.82970020400792e-05, "loss": 0.019294973462820053, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01948, "step": 223, "tokens/total": 7219024, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 101351} +{"epoch": 0.875, "grad_norm": 0.3828311860561371, "learning_rate": 6.801939899284132e-05, "loss": 0.008842012844979763, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00888, "step": 224, "tokens/total": 7253816, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 101857} +{"epoch": 0.87890625, "grad_norm": 0.14722862839698792, "learning_rate": 6.774125415526827e-05, "loss": 0.002048286609351635, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00205, "step": 225, "tokens/total": 7286344, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 102328} +{"epoch": 0.8828125, "grad_norm": 0.3862244784832001, "learning_rate": 6.746257910210214e-05, "loss": 0.007296768948435783, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00732, "step": 226, "tokens/total": 7317952, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 102779} +{"epoch": 0.88671875, "grad_norm": 0.9387543201446533, "learning_rate": 6.718338543014937e-05, "loss": 0.005068998783826828, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00508, "step": 227, "tokens/total": 7347336, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 103228} +{"epoch": 0.890625, "grad_norm": 0.07507016509771347, "learning_rate": 6.69036847577983e-05, "loss": 0.0017531684134155512, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00175, "step": 228, "tokens/total": 7377592, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 103684} +{"epoch": 0.89453125, "grad_norm": 19.453468322753906, "learning_rate": 6.662348872453553e-05, "loss": 0.008416872471570969, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00845, "step": 229, "tokens/total": 7410184, "tokens/train_per_sec_per_gpu": 10.47, "tokens/trainable": 104145} +{"epoch": 0.8984375, "grad_norm": 0.30000606179237366, "learning_rate": 6.63428089904618e-05, "loss": 0.004683700390160084, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00469, "step": 230, "tokens/total": 7443984, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 104598} +{"epoch": 0.90234375, "grad_norm": 0.1989770084619522, "learning_rate": 6.60616572358065e-05, "loss": 0.003345719538629055, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00335, "step": 231, "tokens/total": 7472480, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 104998} +{"epoch": 0.90625, "grad_norm": 0.09697987884283066, "learning_rate": 6.578004516044172e-05, "loss": 0.0021666118409484625, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00217, "step": 232, "tokens/total": 7505992, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 105465} +{"epoch": 0.91015625, "grad_norm": 0.24961699545383453, "learning_rate": 6.549798448339548e-05, "loss": 0.004082606174051762, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00409, "step": 233, "tokens/total": 7541168, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 105928} +{"epoch": 0.9140625, "grad_norm": 0.12049819529056549, "learning_rate": 6.521548694236384e-05, "loss": 0.0011472483165562153, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00115, "step": 234, "tokens/total": 7574208, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 106365} +{"epoch": 0.91796875, "grad_norm": 0.17926888167858124, "learning_rate": 6.493256429322259e-05, "loss": 0.004981872625648975, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00499, "step": 235, "tokens/total": 7607016, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 106796} +{"epoch": 0.921875, "grad_norm": 0.7208530902862549, "learning_rate": 6.464922830953799e-05, "loss": 0.014821010641753674, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.01493, "step": 236, "tokens/total": 7641744, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 107227} +{"epoch": 0.92578125, "grad_norm": 0.8236607313156128, "learning_rate": 6.436549078207688e-05, "loss": 0.014367003925144672, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.01447, "step": 237, "tokens/total": 7673096, "tokens/train_per_sec_per_gpu": 10.09, "tokens/trainable": 107687} +{"epoch": 0.9296875, "grad_norm": 0.4145333170890808, "learning_rate": 6.408136351831592e-05, "loss": 0.010023444890975952, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.01007, "step": 238, "tokens/total": 7705344, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 108127} +{"epoch": 0.93359375, "grad_norm": 0.054829277098178864, "learning_rate": 6.379685834195036e-05, "loss": 0.000892829557415098, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00089, "step": 239, "tokens/total": 7740240, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 108633} +{"epoch": 0.9375, "grad_norm": 0.2571236789226532, "learning_rate": 6.351198709240186e-05, "loss": 0.0039504533633589745, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00396, "step": 240, "tokens/total": 7775392, "tokens/train_per_sec_per_gpu": 7.0, "tokens/trainable": 109085} +{"epoch": 0.94140625, "grad_norm": 0.2263481169939041, "learning_rate": 6.32267616243259e-05, "loss": 0.004340440034866333, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00435, "step": 241, "tokens/total": 7807296, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 109548} +{"epoch": 0.9453125, "grad_norm": 0.2000426948070526, "learning_rate": 6.294119380711849e-05, "loss": 0.0015298149082809687, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00153, "step": 242, "tokens/total": 7838200, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 109975} +{"epoch": 0.94921875, "grad_norm": 2.06322979927063, "learning_rate": 6.265529552442209e-05, "loss": 0.022546270862221718, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.0228, "step": 243, "tokens/total": 7870936, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 110436} +{"epoch": 0.953125, "grad_norm": 0.4275593161582947, "learning_rate": 6.236907867363127e-05, "loss": 0.023255042731761932, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.02353, "step": 244, "tokens/total": 7902512, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 110882} +{"epoch": 0.95703125, "grad_norm": 0.23420189321041107, "learning_rate": 6.208255516539749e-05, "loss": 0.004598200786858797, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00461, "step": 245, "tokens/total": 7933224, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 111355} +{"epoch": 0.9609375, "grad_norm": 0.022161617875099182, "learning_rate": 6.179573692313344e-05, "loss": 0.0003301772812847048, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00033, "step": 246, "tokens/total": 7968456, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 111803} +{"epoch": 0.96484375, "grad_norm": 0.07838483154773712, "learning_rate": 6.150863588251694e-05, "loss": 0.0014561382122337818, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00146, "step": 247, "tokens/total": 8003592, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 112270} +{"epoch": 0.96875, "grad_norm": 0.33865034580230713, "learning_rate": 6.122126399099419e-05, "loss": 0.007855161093175411, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00789, "step": 248, "tokens/total": 8040048, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 112778} +{"epoch": 0.97265625, "grad_norm": 9.639228820800781, "learning_rate": 6.0933633207282615e-05, "loss": 0.013926341198384762, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.01402, "step": 249, "tokens/total": 8074000, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 113284} +{"epoch": 0.9765625, "grad_norm": 0.13091161847114563, "learning_rate": 6.064575550087316e-05, "loss": 0.0032837993931025267, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00329, "step": 250, "tokens/total": 8108304, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 113722} +{"epoch": 0.98046875, "grad_norm": 0.1779211312532425, "learning_rate": 6.0357642851532245e-05, "loss": 0.004718167241662741, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00473, "step": 251, "tokens/total": 8143464, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 114213} +{"epoch": 0.984375, "grad_norm": 0.1602438986301422, "learning_rate": 6.0069307248803294e-05, "loss": 0.002952839247882366, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00296, "step": 252, "tokens/total": 8175576, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 114634} +{"epoch": 0.98828125, "grad_norm": 0.37795397639274597, "learning_rate": 5.9780760691507635e-05, "loss": 0.010825985111296177, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.01088, "step": 253, "tokens/total": 8205528, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 115066} +{"epoch": 0.9921875, "grad_norm": 0.3673722445964813, "learning_rate": 5.9492015187245334e-05, "loss": 0.00863550789654255, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00867, "step": 254, "tokens/total": 8239424, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 115520} +{"epoch": 0.99609375, "grad_norm": 0.16659584641456604, "learning_rate": 5.920308275189541e-05, "loss": 0.0026361090131103992, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00264, "step": 255, "tokens/total": 8270392, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 115921} +{"epoch": 1.0, "grad_norm": 0.3864063620567322, "learning_rate": 5.8913975409115874e-05, "loss": 0.015075980685651302, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.01519, "step": 256, "tokens/total": 8302096, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 116384} +{"epoch": 1.00390625, "grad_norm": 0.1708451509475708, "learning_rate": 5.8624705189843395e-05, "loss": 0.002032655291259289, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00203, "step": 257, "tokens/total": 8337064, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 116871} +{"epoch": 1.0078125, "grad_norm": 0.13460533320903778, "learning_rate": 5.833528413179249e-05, "loss": 0.0018581849290058017, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00186, "step": 258, "tokens/total": 8372840, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 117369} +{"epoch": 1.01171875, "grad_norm": 0.25189730525016785, "learning_rate": 5.80457242789548e-05, "loss": 0.005085050594061613, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.0051, "step": 259, "tokens/total": 8409728, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 117866} +{"epoch": 1.015625, "grad_norm": 0.2457781434059143, "learning_rate": 5.77560376810977e-05, "loss": 0.007704057265073061, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.00773, "step": 260, "tokens/total": 8442296, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 118330} +{"epoch": 1.01953125, "grad_norm": 0.05706380307674408, "learning_rate": 5.7466236393263005e-05, "loss": 0.00065721565624699, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00066, "step": 261, "tokens/total": 8474568, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 118776} +{"epoch": 1.0234375, "grad_norm": 0.016687408089637756, "learning_rate": 5.717633247526522e-05, "loss": 0.0003916716668754816, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00039, "step": 262, "tokens/total": 8505520, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 119211} +{"epoch": 1.02734375, "grad_norm": 0.1160297691822052, "learning_rate": 5.688633799118971e-05, "loss": 0.0013183465925976634, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00132, "step": 263, "tokens/total": 8535064, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 119616} +{"epoch": 1.03125, "grad_norm": 0.027115851640701294, "learning_rate": 5.659626500889066e-05, "loss": 0.000446373742306605, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00045, "step": 264, "tokens/total": 8563600, "tokens/train_per_sec_per_gpu": 6.99, "tokens/trainable": 120086} +{"epoch": 1.03515625, "grad_norm": 0.010182307101786137, "learning_rate": 5.6306125599488905e-05, "loss": 0.0001881028147181496, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00019, "step": 265, "tokens/total": 8592648, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 120525} +{"epoch": 1.0390625, "grad_norm": 0.21027396619319916, "learning_rate": 5.601593183686955e-05, "loss": 0.00338816293515265, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00339, "step": 266, "tokens/total": 8627328, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 121001} +{"epoch": 1.04296875, "grad_norm": 0.2516525983810425, "learning_rate": 5.572569579717961e-05, "loss": 0.0028036613948643208, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00281, "step": 267, "tokens/total": 8659760, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 121465} +{"epoch": 1.046875, "grad_norm": 0.01695103757083416, "learning_rate": 5.543542955832538e-05, "loss": 0.00016218118253163993, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00016, "step": 268, "tokens/total": 8696968, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 121908} +{"epoch": 1.05078125, "grad_norm": 0.1362559199333191, "learning_rate": 5.514514519946986e-05, "loss": 0.0019225262803956866, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00192, "step": 269, "tokens/total": 8735392, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 122406} +{"epoch": 1.0546875, "grad_norm": 0.018645865842700005, "learning_rate": 5.485485480053015e-05, "loss": 0.0002794744505081326, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00028, "step": 270, "tokens/total": 8769840, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 122812} +{"epoch": 1.05859375, "grad_norm": 0.06814008206129074, "learning_rate": 5.4564570441674645e-05, "loss": 0.0008602809975855052, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00086, "step": 271, "tokens/total": 8800648, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 123252} +{"epoch": 1.0625, "grad_norm": 0.018192078918218613, "learning_rate": 5.42743042028204e-05, "loss": 0.00027617713203653693, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00028, "step": 272, "tokens/total": 8832480, "tokens/train_per_sec_per_gpu": 6.84, "tokens/trainable": 123692} +{"epoch": 1.06640625, "grad_norm": 0.06721559911966324, "learning_rate": 5.3984068163130464e-05, "loss": 0.00042911231867037714, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00043, "step": 273, "tokens/total": 8864840, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 124162} +{"epoch": 1.0703125, "grad_norm": 0.08861538767814636, "learning_rate": 5.369387440051111e-05, "loss": 0.0006796877714805305, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00068, "step": 274, "tokens/total": 8897272, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 124625} +{"epoch": 1.07421875, "grad_norm": 0.002867969684302807, "learning_rate": 5.340373499110935e-05, "loss": 4.8178542783716694e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00005, "step": 275, "tokens/total": 8930688, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 125119} +{"epoch": 1.078125, "grad_norm": 0.0513216070830822, "learning_rate": 5.3113662008810304e-05, "loss": 0.0004031726566608995, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.0004, "step": 276, "tokens/total": 8967216, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 125601} +{"epoch": 1.08203125, "grad_norm": 0.2174728959798813, "learning_rate": 5.282366752473479e-05, "loss": 0.0017823673551902175, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00178, "step": 277, "tokens/total": 9001368, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 126075} +{"epoch": 1.0859375, "grad_norm": 0.6591933369636536, "learning_rate": 5.2533763606737005e-05, "loss": 0.006180719006806612, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.0062, "step": 278, "tokens/total": 9036808, "tokens/train_per_sec_per_gpu": 6.86, "tokens/trainable": 126491} +{"epoch": 1.08984375, "grad_norm": 0.004069754388183355, "learning_rate": 5.224396231890232e-05, "loss": 6.403612496796995e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00006, "step": 279, "tokens/total": 9067840, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 126940} +{"epoch": 1.09375, "grad_norm": 0.027886807918548584, "learning_rate": 5.195427572104522e-05, "loss": 0.0002594169636722654, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00026, "step": 280, "tokens/total": 9098936, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 127376} +{"epoch": 1.09765625, "grad_norm": 0.3329349756240845, "learning_rate": 5.166471586820751e-05, "loss": 0.002998914336785674, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.003, "step": 281, "tokens/total": 9133264, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 127841} +{"epoch": 1.1015625, "grad_norm": 0.4372626841068268, "learning_rate": 5.1375294810156615e-05, "loss": 0.004284100607037544, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00429, "step": 282, "tokens/total": 9165008, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 128288} +{"epoch": 1.10546875, "grad_norm": 0.003697008825838566, "learning_rate": 5.1086024590884144e-05, "loss": 5.207298818277195e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00005, "step": 283, "tokens/total": 9197952, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 128737} +{"epoch": 1.109375, "grad_norm": 0.23917244374752045, "learning_rate": 5.079691724810461e-05, "loss": 0.001817085430957377, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00182, "step": 284, "tokens/total": 9232576, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 129172} +{"epoch": 1.11328125, "grad_norm": 0.5931620001792908, "learning_rate": 5.0507984812754684e-05, "loss": 0.011604126542806625, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.01167, "step": 285, "tokens/total": 9264568, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 129669} +{"epoch": 1.1171875, "grad_norm": 0.09573046118021011, "learning_rate": 5.021923930849237e-05, "loss": 0.0009820424020290375, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00098, "step": 286, "tokens/total": 9296544, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 130155} +{"epoch": 1.12109375, "grad_norm": 0.21048255264759064, "learning_rate": 4.99306927511967e-05, "loss": 0.0021737003698945045, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00218, "step": 287, "tokens/total": 9327936, "tokens/train_per_sec_per_gpu": 10.36, "tokens/trainable": 130595} +{"epoch": 1.125, "grad_norm": 0.05695793405175209, "learning_rate": 4.964235714846775e-05, "loss": 0.0004605480935424566, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00046, "step": 288, "tokens/total": 9357984, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 131015} +{"epoch": 1.12890625, "grad_norm": 0.007052319124341011, "learning_rate": 4.9354244499126866e-05, "loss": 8.513370266882703e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00009, "step": 289, "tokens/total": 9388664, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 131481} +{"epoch": 1.1328125, "grad_norm": 0.1312360316514969, "learning_rate": 4.90663667927174e-05, "loss": 0.0012757983058691025, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00128, "step": 290, "tokens/total": 9420504, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 131930} +{"epoch": 1.13671875, "grad_norm": 0.044005151838064194, "learning_rate": 4.877873600900581e-05, "loss": 0.0004971766611561179, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.0005, "step": 291, "tokens/total": 9452664, "tokens/train_per_sec_per_gpu": 6.18, "tokens/trainable": 132388} +{"epoch": 1.140625, "grad_norm": 0.03776595741510391, "learning_rate": 4.849136411748306e-05, "loss": 0.0002033245109487325, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.0002, "step": 292, "tokens/total": 9488896, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 132860} +{"epoch": 1.14453125, "grad_norm": 0.005746347364038229, "learning_rate": 4.8204263076866574e-05, "loss": 6.452562229242176e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00006, "step": 293, "tokens/total": 9522112, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 133360} +{"epoch": 1.1484375, "grad_norm": 0.04935140535235405, "learning_rate": 4.791744483460251e-05, "loss": 0.00022254370560403913, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00022, "step": 294, "tokens/total": 9555632, "tokens/train_per_sec_per_gpu": 6.55, "tokens/trainable": 133793} +{"epoch": 1.15234375, "grad_norm": 0.004607086069881916, "learning_rate": 4.7630921326368736e-05, "loss": 5.457473889691755e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00005, "step": 295, "tokens/total": 9590904, "tokens/train_per_sec_per_gpu": 6.39, "tokens/trainable": 134218} +{"epoch": 1.15625, "grad_norm": 0.048835981637239456, "learning_rate": 4.7344704475577916e-05, "loss": 0.0003682714013848454, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00037, "step": 296, "tokens/total": 9620288, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 134654} +{"epoch": 1.16015625, "grad_norm": 0.4510250389575958, "learning_rate": 4.705880619288153e-05, "loss": 0.011205228976905346, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.01127, "step": 297, "tokens/total": 9651984, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 135056} +{"epoch": 1.1640625, "grad_norm": 0.40895703434944153, "learning_rate": 4.677323837567412e-05, "loss": 0.011390063911676407, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.01146, "step": 298, "tokens/total": 9681368, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 135483} +{"epoch": 1.16796875, "grad_norm": 0.004125889856368303, "learning_rate": 4.6488012907598146e-05, "loss": 6.274677434703335e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00006, "step": 299, "tokens/total": 9712872, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 135957} +{"epoch": 1.171875, "grad_norm": 0.045019809156656265, "learning_rate": 4.620314165804964e-05, "loss": 0.00044594091014005244, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00045, "step": 300, "tokens/total": 9745184, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 136430} +{"epoch": 1.17578125, "grad_norm": 0.22345896065235138, "learning_rate": 4.591863648168407e-05, "loss": 0.0018077562563121319, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00181, "step": 301, "tokens/total": 9778928, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 136896} +{"epoch": 1.1796875, "grad_norm": 0.2476399689912796, "learning_rate": 4.5634509217923135e-05, "loss": 0.002283818554133177, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00229, "step": 302, "tokens/total": 9810752, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 137331} +{"epoch": 1.18359375, "grad_norm": 0.1829226166009903, "learning_rate": 4.535077169046201e-05, "loss": 0.0013085578102618456, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00131, "step": 303, "tokens/total": 9843672, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 137768} +{"epoch": 1.1875, "grad_norm": 0.7799065113067627, "learning_rate": 4.506743570677743e-05, "loss": 0.007660662289708853, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00769, "step": 304, "tokens/total": 9873752, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 138200} +{"epoch": 1.19140625, "grad_norm": 0.2760083079338074, "learning_rate": 4.478451305763618e-05, "loss": 0.005876438692212105, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00589, "step": 305, "tokens/total": 9906272, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 138660} +{"epoch": 1.1953125, "grad_norm": 0.0849539265036583, "learning_rate": 4.450201551660454e-05, "loss": 0.0003704153059516102, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00037, "step": 306, "tokens/total": 9936296, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 139092} +{"epoch": 1.19921875, "grad_norm": 0.08318056911230087, "learning_rate": 4.4219954839558276e-05, "loss": 0.0010343438480049372, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00103, "step": 307, "tokens/total": 9967720, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 139533} +{"epoch": 1.203125, "grad_norm": 0.6498168706893921, "learning_rate": 4.393834276419352e-05, "loss": 0.007257598917931318, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.00728, "step": 308, "tokens/total": 9997176, "tokens/train_per_sec_per_gpu": 10.73, "tokens/trainable": 140020} +{"epoch": 1.20703125, "grad_norm": 0.2268553227186203, "learning_rate": 4.36571910095382e-05, "loss": 0.0003216843761038035, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00032, "step": 309, "tokens/total": 10031872, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 140465} +{"epoch": 1.2109375, "grad_norm": 0.013355248607695103, "learning_rate": 4.337651127546448e-05, "loss": 0.000235057930694893, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00024, "step": 310, "tokens/total": 10065240, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 140915} +{"epoch": 1.21484375, "grad_norm": 0.03532262519001961, "learning_rate": 4.3096315242201736e-05, "loss": 0.0005856506759300828, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00059, "step": 311, "tokens/total": 10096640, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 141350} +{"epoch": 1.21875, "grad_norm": 0.004510571248829365, "learning_rate": 4.2816614569850635e-05, "loss": 4.505493416218087e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00005, "step": 312, "tokens/total": 10130192, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 141789} +{"epoch": 1.22265625, "grad_norm": 0.49557366967201233, "learning_rate": 4.2537420897897864e-05, "loss": 0.004607683978974819, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00462, "step": 313, "tokens/total": 10164248, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 142266} +{"epoch": 1.2265625, "grad_norm": 0.05606323480606079, "learning_rate": 4.225874584473174e-05, "loss": 0.001045818324200809, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00105, "step": 314, "tokens/total": 10194664, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 142704} +{"epoch": 1.23046875, "grad_norm": 0.01274903118610382, "learning_rate": 4.19806010071587e-05, "loss": 8.563503797631711e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00009, "step": 315, "tokens/total": 10227768, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 143168} +{"epoch": 1.234375, "grad_norm": 0.04427577927708626, "learning_rate": 4.170299795992081e-05, "loss": 0.0004864969232585281, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00049, "step": 316, "tokens/total": 10260464, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 143615} +{"epoch": 1.23828125, "grad_norm": 0.2213975042104721, "learning_rate": 4.142594825521398e-05, "loss": 0.004006124567240477, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00401, "step": 317, "tokens/total": 10297528, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 144096} +{"epoch": 1.2421875, "grad_norm": 0.001942704780958593, "learning_rate": 4.114946342220728e-05, "loss": 3.519464735290967e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00004, "step": 318, "tokens/total": 10331448, "tokens/train_per_sec_per_gpu": 6.95, "tokens/trainable": 144543} +{"epoch": 1.24609375, "grad_norm": 0.10148075968027115, "learning_rate": 4.087355496656321e-05, "loss": 0.0006823099683970213, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00068, "step": 319, "tokens/total": 10363152, "tokens/train_per_sec_per_gpu": 10.09, "tokens/trainable": 145016} +{"epoch": 1.25, "grad_norm": 0.05805341154336929, "learning_rate": 4.05982343699588e-05, "loss": 0.0009420415153726935, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00094, "step": 320, "tokens/total": 10397832, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 145511} +{"epoch": 1.25390625, "grad_norm": 0.07461350411176682, "learning_rate": 4.0323513089607876e-05, "loss": 0.001237902557477355, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00124, "step": 321, "tokens/total": 10431416, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 145978} +{"epoch": 1.2578125, "grad_norm": 0.49324819445610046, "learning_rate": 4.004940255778431e-05, "loss": 0.0021292041055858135, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00213, "step": 322, "tokens/total": 10467384, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 146422} +{"epoch": 1.26171875, "grad_norm": 0.015673978254199028, "learning_rate": 3.977591418134619e-05, "loss": 0.0001598229428054765, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00016, "step": 323, "tokens/total": 10500792, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 146863} +{"epoch": 1.265625, "grad_norm": 0.09156472980976105, "learning_rate": 3.95030593412612e-05, "loss": 0.0008779853815212846, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00088, "step": 324, "tokens/total": 10532592, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 147286} +{"epoch": 1.26953125, "grad_norm": 0.015417955815792084, "learning_rate": 3.923084939213296e-05, "loss": 0.00021575044956989586, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00022, "step": 325, "tokens/total": 10562392, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 147708} +{"epoch": 1.2734375, "grad_norm": 0.1811457872390747, "learning_rate": 3.895929566172861e-05, "loss": 0.0007246144232340157, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00072, "step": 326, "tokens/total": 10595416, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 148172} +{"epoch": 1.27734375, "grad_norm": 0.005374716594815254, "learning_rate": 3.868840945050728e-05, "loss": 6.295726052485406e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00006, "step": 327, "tokens/total": 10630504, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 148604} +{"epoch": 1.28125, "grad_norm": 0.0952359065413475, "learning_rate": 3.841820203114995e-05, "loss": 0.00043539507896639407, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00044, "step": 328, "tokens/total": 10661632, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 149051} +{"epoch": 1.28515625, "grad_norm": 0.0656227394938469, "learning_rate": 3.814868464809027e-05, "loss": 0.0004846511292271316, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00048, "step": 329, "tokens/total": 10691544, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 149453} +{"epoch": 1.2890625, "grad_norm": 0.09828502684831619, "learning_rate": 3.787986851704667e-05, "loss": 0.0008396340417675674, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00084, "step": 330, "tokens/total": 10724768, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 149883} +{"epoch": 1.29296875, "grad_norm": 0.00583867821842432, "learning_rate": 3.7611764824555654e-05, "loss": 6.288621807470918e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00006, "step": 331, "tokens/total": 10756008, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 150357} +{"epoch": 1.296875, "grad_norm": 0.004756614100188017, "learning_rate": 3.734438472750619e-05, "loss": 4.3227100832154974e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00004, "step": 332, "tokens/total": 10793152, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 150828} +{"epoch": 1.30078125, "grad_norm": 0.2174200415611267, "learning_rate": 3.707773935267552e-05, "loss": 0.0013138551730662584, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00131, "step": 333, "tokens/total": 10827624, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 151314} +{"epoch": 1.3046875, "grad_norm": 0.0013699992559850216, "learning_rate": 3.68118397962661e-05, "loss": 1.539649565529544e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00002, "step": 334, "tokens/total": 10859712, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 151722} +{"epoch": 1.30859375, "grad_norm": 0.027371220290660858, "learning_rate": 3.654669712344384e-05, "loss": 0.00011474905477371067, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00011, "step": 335, "tokens/total": 10888792, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 152178} +{"epoch": 1.3125, "grad_norm": 0.0531560443341732, "learning_rate": 3.628232236787763e-05, "loss": 0.00036535595427267253, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00037, "step": 336, "tokens/total": 10922968, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 152617} +{"epoch": 1.31640625, "grad_norm": 0.2685186266899109, "learning_rate": 3.6018726531280144e-05, "loss": 0.0011674549896270037, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00117, "step": 337, "tokens/total": 10958000, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 153108} +{"epoch": 1.3203125, "grad_norm": 0.3160148859024048, "learning_rate": 3.575592058295017e-05, "loss": 0.0025059590116143227, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00251, "step": 338, "tokens/total": 10989872, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 153589} +{"epoch": 1.32421875, "grad_norm": 0.0007089831633493304, "learning_rate": 3.549391545931585e-05, "loss": 6.3551487983204424e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00001, "step": 339, "tokens/total": 11020784, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 154049} +{"epoch": 1.328125, "grad_norm": 0.0012141778133809566, "learning_rate": 3.5232722063479914e-05, "loss": 7.854447176214308e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00001, "step": 340, "tokens/total": 11053240, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 154498} +{"epoch": 1.33203125, "grad_norm": 0.025186441838741302, "learning_rate": 3.49723512647657e-05, "loss": 0.00019170103769283742, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00019, "step": 341, "tokens/total": 11089040, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 154998} +{"epoch": 1.3359375, "grad_norm": 0.0005151353543624282, "learning_rate": 3.471281389826491e-05, "loss": 9.433420018467586e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00001, "step": 342, "tokens/total": 11119672, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 155403} +{"epoch": 1.33984375, "grad_norm": 0.005653131287544966, "learning_rate": 3.4454120764386764e-05, "loss": 1.7694623238639906e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00002, "step": 343, "tokens/total": 11155040, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 155858} +{"epoch": 1.34375, "grad_norm": 0.08691707998514175, "learning_rate": 3.4196282628408526e-05, "loss": 0.0008413225295953453, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00084, "step": 344, "tokens/total": 11185952, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 156303} +{"epoch": 1.34765625, "grad_norm": 0.08739098906517029, "learning_rate": 3.3939310220027456e-05, "loss": 0.0001753513206494972, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00018, "step": 345, "tokens/total": 11219272, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 156798} +{"epoch": 1.3515625, "grad_norm": 0.000682034355122596, "learning_rate": 3.3683214232914404e-05, "loss": 7.275975804077461e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00001, "step": 346, "tokens/total": 11253424, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 157271} +{"epoch": 1.35546875, "grad_norm": 0.34361937642097473, "learning_rate": 3.342800532426873e-05, "loss": 0.003305097110569477, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00331, "step": 347, "tokens/total": 11286216, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 157737} +{"epoch": 1.359375, "grad_norm": 0.06034113094210625, "learning_rate": 3.317369411437484e-05, "loss": 0.00036789008299820125, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00037, "step": 348, "tokens/total": 11319480, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 158205} +{"epoch": 1.36328125, "grad_norm": 0.008653691969811916, "learning_rate": 3.292029118616024e-05, "loss": 2.9611734134959988e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00003, "step": 349, "tokens/total": 11350472, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 158628} +{"epoch": 1.3671875, "grad_norm": 0.0019191160099580884, "learning_rate": 3.266780708475511e-05, "loss": 2.076571763609536e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00002, "step": 350, "tokens/total": 11383088, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 159106} +{"epoch": 1.37109375, "grad_norm": 0.000917036144528538, "learning_rate": 3.241625231705354e-05, "loss": 1.1268572961853351e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00001, "step": 351, "tokens/total": 11418392, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 159565} +{"epoch": 1.375, "grad_norm": 0.08271697908639908, "learning_rate": 3.216563735127618e-05, "loss": 0.000506524695083499, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00051, "step": 352, "tokens/total": 11449784, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 160042} +{"epoch": 1.37890625, "grad_norm": 0.06369170546531677, "learning_rate": 3.191597261653475e-05, "loss": 0.00020022239186801016, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.0002, "step": 353, "tokens/total": 11484240, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 160502} +{"epoch": 1.3828125, "grad_norm": 0.023014133796095848, "learning_rate": 3.166726850239794e-05, "loss": 9.399090049555525e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00009, "step": 354, "tokens/total": 11513360, "tokens/train_per_sec_per_gpu": 10.94, "tokens/trainable": 160950} +{"epoch": 1.38671875, "grad_norm": 0.0004683267616201192, "learning_rate": 3.141953535845912e-05, "loss": 3.394015038793441e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.0, "step": 355, "tokens/total": 11543112, "tokens/train_per_sec_per_gpu": 10.2, "tokens/trainable": 161400} +{"epoch": 1.390625, "grad_norm": 0.00521289324387908, "learning_rate": 3.11727834939056e-05, "loss": 4.847055970458314e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00005, "step": 356, "tokens/total": 11573464, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 161863} +{"epoch": 1.39453125, "grad_norm": 0.3214205205440521, "learning_rate": 3.092702317708967e-05, "loss": 0.00788242369890213, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00791, "step": 357, "tokens/total": 11604896, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 162319} +{"epoch": 1.3984375, "grad_norm": 0.003189283423125744, "learning_rate": 3.0682264635101276e-05, "loss": 1.5024528693174943e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00002, "step": 358, "tokens/total": 11636368, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 162771} +{"epoch": 1.40234375, "grad_norm": 0.44971325993537903, "learning_rate": 3.0438518053342407e-05, "loss": 0.002231946447864175, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00223, "step": 359, "tokens/total": 11670216, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 163243} +{"epoch": 1.40625, "grad_norm": 0.3163140118122101, "learning_rate": 3.0195793575103266e-05, "loss": 0.0014030667953193188, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.0014, "step": 360, "tokens/total": 11707816, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 163707} +{"epoch": 1.41015625, "grad_norm": 4.428566932678223, "learning_rate": 2.9954101301140146e-05, "loss": 0.005831625312566757, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00585, "step": 361, "tokens/total": 11742584, "tokens/train_per_sec_per_gpu": 6.86, "tokens/trainable": 164157} +{"epoch": 1.4140625, "grad_norm": 0.004845458082854748, "learning_rate": 2.9713451289255123e-05, "loss": 5.101397255202755e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00005, "step": 362, "tokens/total": 11770384, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 164598} +{"epoch": 1.41796875, "grad_norm": 0.00046998882316984236, "learning_rate": 2.9473853553877484e-05, "loss": 5.7022052715183236e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00001, "step": 363, "tokens/total": 11803208, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 165072} +{"epoch": 1.421875, "grad_norm": 0.03452037647366524, "learning_rate": 2.9235318065647e-05, "loss": 0.00016167649300768971, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00016, "step": 364, "tokens/total": 11840480, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 165533} +{"epoch": 1.42578125, "grad_norm": 0.0013538396451622248, "learning_rate": 2.8997854750998964e-05, "loss": 1.1396510672057047e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00001, "step": 365, "tokens/total": 11875544, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 165998} +{"epoch": 1.4296875, "grad_norm": 0.0002057611709460616, "learning_rate": 2.8761473491751258e-05, "loss": 3.366978944541188e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.0, "step": 366, "tokens/total": 11907656, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 166433} +{"epoch": 1.43359375, "grad_norm": 0.019655374810099602, "learning_rate": 2.8526184124692883e-05, "loss": 4.186746446066536e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00004, "step": 367, "tokens/total": 11941904, "tokens/train_per_sec_per_gpu": 6.37, "tokens/trainable": 166871} +{"epoch": 1.4375, "grad_norm": 0.010770595632493496, "learning_rate": 2.829199644117484e-05, "loss": 0.00013474309525918216, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00013, "step": 368, "tokens/total": 11973624, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 167281} +{"epoch": 1.44140625, "grad_norm": 0.008004900999367237, "learning_rate": 2.8058920186702553e-05, "loss": 6.648715498158708e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00007, "step": 369, "tokens/total": 12010360, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 167754} +{"epoch": 1.4453125, "grad_norm": 0.012337073683738708, "learning_rate": 2.782696506053033e-05, "loss": 7.163315603975207e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00007, "step": 370, "tokens/total": 12044544, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 168186} +{"epoch": 1.44921875, "grad_norm": 0.06365065276622772, "learning_rate": 2.7596140715257824e-05, "loss": 0.0004978164215572178, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.0005, "step": 371, "tokens/total": 12075792, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 168651} +{"epoch": 1.453125, "grad_norm": 0.00036956032272428274, "learning_rate": 2.7366456756428184e-05, "loss": 5.028933628636878e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00001, "step": 372, "tokens/total": 12108088, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 169108} +{"epoch": 1.45703125, "grad_norm": 0.001114205806516111, "learning_rate": 2.7137922742128486e-05, "loss": 1.0725513675424736e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00001, "step": 373, "tokens/total": 12140888, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 169607} +{"epoch": 1.4609375, "grad_norm": 0.05998492240905762, "learning_rate": 2.691054818259188e-05, "loss": 0.0004106312117073685, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00041, "step": 374, "tokens/total": 12174560, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 170080} +{"epoch": 1.46484375, "grad_norm": 0.27520981431007385, "learning_rate": 2.6684342539801933e-05, "loss": 0.0028842820320278406, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00289, "step": 375, "tokens/total": 12204848, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 170532} +{"epoch": 1.46875, "grad_norm": 0.014789705164730549, "learning_rate": 2.645931522709877e-05, "loss": 8.107903704512864e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00008, "step": 376, "tokens/total": 12238472, "tokens/train_per_sec_per_gpu": 10.7, "tokens/trainable": 171012} +{"epoch": 1.47265625, "grad_norm": 0.033284496515989304, "learning_rate": 2.6235475608787365e-05, "loss": 0.0001306091871811077, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00013, "step": 377, "tokens/total": 12271024, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 171473} +{"epoch": 1.4765625, "grad_norm": 0.0021434673108160496, "learning_rate": 2.6012832999747916e-05, "loss": 2.8939823096152395e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00003, "step": 378, "tokens/total": 12301312, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 171920} +{"epoch": 1.48046875, "grad_norm": 0.5366498827934265, "learning_rate": 2.579139666504821e-05, "loss": 0.015690285712480545, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01581, "step": 379, "tokens/total": 12337072, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 172366} +{"epoch": 1.484375, "grad_norm": 0.003966609016060829, "learning_rate": 2.557117581955798e-05, "loss": 3.1990355637390167e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00003, "step": 380, "tokens/total": 12368000, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 172832} +{"epoch": 1.48828125, "grad_norm": 0.0021749227307736874, "learning_rate": 2.5352179627565532e-05, "loss": 3.4202224924229085e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00003, "step": 381, "tokens/total": 12399880, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 173272} +{"epoch": 1.4921875, "grad_norm": 0.00039216247387230396, "learning_rate": 2.5134417202396277e-05, "loss": 6.4203350120806135e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00001, "step": 382, "tokens/total": 12432288, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 173679} +{"epoch": 1.49609375, "grad_norm": 0.13038349151611328, "learning_rate": 2.491789760603361e-05, "loss": 0.0008726094965822995, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00087, "step": 383, "tokens/total": 12465592, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 174109} +{"epoch": 1.5, "grad_norm": 0.014167835004627705, "learning_rate": 2.4702629848741764e-05, "loss": 0.00011326140520395711, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00011, "step": 384, "tokens/total": 12498200, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 174564} +{"epoch": 1.50390625, "grad_norm": 0.4452621638774872, "learning_rate": 2.4488622888690785e-05, "loss": 0.008744060061872005, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00878, "step": 385, "tokens/total": 12530280, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 175004} +{"epoch": 1.5078125, "grad_norm": 0.004893981385976076, "learning_rate": 2.427588563158384e-05, "loss": 4.681666905526072e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00005, "step": 386, "tokens/total": 12563360, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 175490} +{"epoch": 1.51171875, "grad_norm": 0.021767526865005493, "learning_rate": 2.406442693028651e-05, "loss": 0.00021483884484041482, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00021, "step": 387, "tokens/total": 12593032, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 175911} +{"epoch": 1.515625, "grad_norm": 0.001951285288669169, "learning_rate": 2.3854255584458547e-05, "loss": 3.291998291388154e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00003, "step": 388, "tokens/total": 12628272, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 176338} +{"epoch": 1.51953125, "grad_norm": 0.021248333156108856, "learning_rate": 2.3645380340187508e-05, "loss": 0.00021728822321165353, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00022, "step": 389, "tokens/total": 12661368, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 176819} +{"epoch": 1.5234375, "grad_norm": 0.24524655938148499, "learning_rate": 2.3437809889624914e-05, "loss": 0.004883044399321079, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.77, "memory/max_allocated (GiB)": 63.77, "ppl": 1.00489, "step": 390, "tokens/total": 12691168, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 177270} +{"epoch": 1.52734375, "grad_norm": 0.0014878148213028908, "learning_rate": 2.3231552870624487e-05, "loss": 2.3999344193725847e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00002, "step": 391, "tokens/total": 12721264, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 177696} +{"epoch": 1.53125, "grad_norm": 0.01138077862560749, "learning_rate": 2.3026617866382657e-05, "loss": 0.00011964394798269495, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00012, "step": 392, "tokens/total": 12753640, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 178193} +{"epoch": 1.53515625, "grad_norm": 0.766298770904541, "learning_rate": 2.2823013405081507e-05, "loss": 0.007235761731863022, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00726, "step": 393, "tokens/total": 12787056, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 178690} +{"epoch": 1.5390625, "grad_norm": 0.0064115701243281364, "learning_rate": 2.2620747959533722e-05, "loss": 5.7874749472830445e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00006, "step": 394, "tokens/total": 12819312, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 179187} +{"epoch": 1.54296875, "grad_norm": 0.16180464625358582, "learning_rate": 2.2419829946830123e-05, "loss": 0.0011929803295060992, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00119, "step": 395, "tokens/total": 12853376, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 179638} +{"epoch": 1.546875, "grad_norm": 0.26372718811035156, "learning_rate": 2.2220267727989325e-05, "loss": 0.001827480737119913, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00183, "step": 396, "tokens/total": 12882960, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 180091} +{"epoch": 1.55078125, "grad_norm": 0.016883326694369316, "learning_rate": 2.202206960760984e-05, "loss": 0.0001540518569527194, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00015, "step": 397, "tokens/total": 12917016, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 180598} +{"epoch": 1.5546875, "grad_norm": 0.006745238788425922, "learning_rate": 2.182524383352446e-05, "loss": 8.310572593472898e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00008, "step": 398, "tokens/total": 12950056, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 181051} +{"epoch": 1.55859375, "grad_norm": 0.32317766547203064, "learning_rate": 2.1629798596457056e-05, "loss": 0.003707478754222393, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00371, "step": 399, "tokens/total": 12981728, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 181501} +{"epoch": 1.5625, "grad_norm": 0.011868590489029884, "learning_rate": 2.1435742029681725e-05, "loss": 0.0001316916022915393, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00013, "step": 400, "tokens/total": 13011464, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 181918} +{"epoch": 1.56640625, "grad_norm": 0.016496237367391586, "learning_rate": 2.124308220868431e-05, "loss": 9.757136285770684e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.0001, "step": 401, "tokens/total": 13044432, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 182373} +{"epoch": 1.5703125, "grad_norm": 0.012939202599227428, "learning_rate": 2.105182715082638e-05, "loss": 4.2795276385731995e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00004, "step": 402, "tokens/total": 13075608, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 182850} +{"epoch": 1.57421875, "grad_norm": 0.012553195469081402, "learning_rate": 2.0861984815011552e-05, "loss": 0.0001103942995541729, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00011, "step": 403, "tokens/total": 13107352, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 183281} +{"epoch": 1.578125, "grad_norm": 0.11848214268684387, "learning_rate": 2.0673563101354323e-05, "loss": 0.001148263574577868, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00115, "step": 404, "tokens/total": 13140736, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 183711} +{"epoch": 1.58203125, "grad_norm": 0.005563119426369667, "learning_rate": 2.0486569850851317e-05, "loss": 5.623573815682903e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00006, "step": 405, "tokens/total": 13174040, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 184174} +{"epoch": 1.5859375, "grad_norm": 0.013088063336908817, "learning_rate": 2.0301012845054956e-05, "loss": 0.00020839361241087317, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00021, "step": 406, "tokens/total": 13207232, "tokens/train_per_sec_per_gpu": 6.64, "tokens/trainable": 184589} +{"epoch": 1.58984375, "grad_norm": 0.007564143277704716, "learning_rate": 2.011689980574966e-05, "loss": 4.626755981007591e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00005, "step": 407, "tokens/total": 13241160, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 185009} +{"epoch": 1.59375, "grad_norm": 0.02239709347486496, "learning_rate": 1.993423839463052e-05, "loss": 0.00023580326524097472, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00024, "step": 408, "tokens/total": 13273104, "tokens/train_per_sec_per_gpu": 10.37, "tokens/trainable": 185466} +{"epoch": 1.59765625, "grad_norm": 0.010168829932808876, "learning_rate": 1.975303621298445e-05, "loss": 9.607432730263099e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.0001, "step": 409, "tokens/total": 13308136, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 185918} +{"epoch": 1.6015625, "grad_norm": 0.3189012110233307, "learning_rate": 1.957330080137385e-05, "loss": 0.00286426511593163, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00287, "step": 410, "tokens/total": 13337816, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 186369} +{"epoch": 1.60546875, "grad_norm": 0.03599262237548828, "learning_rate": 1.9395039639322864e-05, "loss": 0.0004907848779112101, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00049, "step": 411, "tokens/total": 13372576, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 186844} +{"epoch": 1.609375, "grad_norm": 0.0018190699629485607, "learning_rate": 1.9218260145006073e-05, "loss": 2.2628624719800428e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00002, "step": 412, "tokens/total": 13406664, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 187330} +{"epoch": 1.61328125, "grad_norm": 0.0030517233535647392, "learning_rate": 1.904296967493982e-05, "loss": 4.173868364887312e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00004, "step": 413, "tokens/total": 13438256, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 187786} +{"epoch": 1.6171875, "grad_norm": 0.25038546323776245, "learning_rate": 1.8869175523676064e-05, "loss": 0.00174095353577286, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00174, "step": 414, "tokens/total": 13472096, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 188281} +{"epoch": 1.62109375, "grad_norm": 0.0018984550843015313, "learning_rate": 1.869688492349885e-05, "loss": 3.4352720831520855e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00003, "step": 415, "tokens/total": 13504192, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 188733} +{"epoch": 1.625, "grad_norm": 0.47127392888069153, "learning_rate": 1.85261050441233e-05, "loss": 0.007550451904535294, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00758, "step": 416, "tokens/total": 13537392, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 189189} +{"epoch": 1.62890625, "grad_norm": 0.2663298547267914, "learning_rate": 1.8356842992397304e-05, "loss": 0.0032004625536501408, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00321, "step": 417, "tokens/total": 13569344, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 189629} +{"epoch": 1.6328125, "grad_norm": 0.27544257044792175, "learning_rate": 1.8189105812005714e-05, "loss": 0.0013918798649683595, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00139, "step": 418, "tokens/total": 13602200, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 190039} +{"epoch": 1.63671875, "grad_norm": 0.005104956682771444, "learning_rate": 1.802290048317732e-05, "loss": 4.707041080109775e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00005, "step": 419, "tokens/total": 13632640, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 190457} +{"epoch": 1.640625, "grad_norm": 0.001118365558795631, "learning_rate": 1.785823392239424e-05, "loss": 2.119855344062671e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00002, "step": 420, "tokens/total": 13664920, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 190898} +{"epoch": 1.64453125, "grad_norm": 0.04072565957903862, "learning_rate": 1.7695112982104225e-05, "loss": 0.00034958706237375736, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00035, "step": 421, "tokens/total": 13697144, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 191337} +{"epoch": 1.6484375, "grad_norm": 0.007015950512140989, "learning_rate": 1.7533544450435433e-05, "loss": 6.626312097068876e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00007, "step": 422, "tokens/total": 13724496, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 191770} +{"epoch": 1.65234375, "grad_norm": 0.004189410712569952, "learning_rate": 1.7373535050913946e-05, "loss": 2.3642660380573943e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00002, "step": 423, "tokens/total": 13757880, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 192207} +{"epoch": 1.65625, "grad_norm": 0.019519543275237083, "learning_rate": 1.721509144218405e-05, "loss": 0.00024748919531702995, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00025, "step": 424, "tokens/total": 13787648, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 192637} +{"epoch": 1.66015625, "grad_norm": 0.007954402826726437, "learning_rate": 1.705822021773101e-05, "loss": 9.35255375225097e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00009, "step": 425, "tokens/total": 13819720, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 193123} +{"epoch": 1.6640625, "grad_norm": 0.24265946447849274, "learning_rate": 1.69029279056068e-05, "loss": 0.0019705151207745075, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00197, "step": 426, "tokens/total": 13854944, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 193564} +{"epoch": 1.66796875, "grad_norm": 0.20565228164196014, "learning_rate": 1.6749220968158415e-05, "loss": 0.003259303281083703, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00326, "step": 427, "tokens/total": 13887824, "tokens/train_per_sec_per_gpu": 10.3, "tokens/trainable": 194070} +{"epoch": 1.671875, "grad_norm": 0.053852710872888565, "learning_rate": 1.659710580175893e-05, "loss": 0.0001423183421138674, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00014, "step": 428, "tokens/total": 13918192, "tokens/train_per_sec_per_gpu": 10.07, "tokens/trainable": 194569} +{"epoch": 1.67578125, "grad_norm": 0.1367453932762146, "learning_rate": 1.644658873654133e-05, "loss": 0.0012113688280805945, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00121, "step": 429, "tokens/total": 13947440, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 195006} +{"epoch": 1.6796875, "grad_norm": 0.001205525011755526, "learning_rate": 1.629767603613508e-05, "loss": 2.237880471511744e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00002, "step": 430, "tokens/total": 13974360, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 195406} +{"epoch": 1.68359375, "grad_norm": 0.05595019459724426, "learning_rate": 1.615037389740547e-05, "loss": 0.0005614347755908966, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00056, "step": 431, "tokens/total": 14006592, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 195850} +{"epoch": 1.6875, "grad_norm": 0.18705418705940247, "learning_rate": 1.600468845019576e-05, "loss": 0.0016364059410989285, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00164, "step": 432, "tokens/total": 14040776, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 196330} +{"epoch": 1.69140625, "grad_norm": 0.0892820730805397, "learning_rate": 1.5860625757072092e-05, "loss": 0.00037253586924634874, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00037, "step": 433, "tokens/total": 14073040, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 196778} +{"epoch": 1.6953125, "grad_norm": 0.00875808298587799, "learning_rate": 1.571819181307116e-05, "loss": 7.297101547010243e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00007, "step": 434, "tokens/total": 14107624, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 197250} +{"epoch": 1.69921875, "grad_norm": 0.07955317944288254, "learning_rate": 1.557739254545075e-05, "loss": 0.0007968397112563252, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.0008, "step": 435, "tokens/total": 14137784, "tokens/train_per_sec_per_gpu": 10.57, "tokens/trainable": 197722} +{"epoch": 1.703125, "grad_norm": 0.037593573331832886, "learning_rate": 1.543823381344311e-05, "loss": 0.0002538833359722048, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00025, "step": 436, "tokens/total": 14167776, "tokens/train_per_sec_per_gpu": 10.69, "tokens/trainable": 198196} +{"epoch": 1.70703125, "grad_norm": 0.3403457999229431, "learning_rate": 1.5300721408011114e-05, "loss": 0.0024053058587014675, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00241, "step": 437, "tokens/total": 14199752, "tokens/train_per_sec_per_gpu": 10.16, "tokens/trainable": 198653} +{"epoch": 1.7109375, "grad_norm": 0.006359902210533619, "learning_rate": 1.5164861051607254e-05, "loss": 7.507679401896894e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.00008, "step": 438, "tokens/total": 14230344, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 199061} +{"epoch": 1.71484375, "grad_norm": 0.004970264621078968, "learning_rate": 1.5030658397935521e-05, "loss": 5.687478187610395e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00006, "step": 439, "tokens/total": 14263808, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 199530} +{"epoch": 1.71875, "grad_norm": 0.18841223418712616, "learning_rate": 1.4898119031716104e-05, "loss": 0.0012898162240162492, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00129, "step": 440, "tokens/total": 14295864, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 199988} +{"epoch": 1.72265625, "grad_norm": 0.020563937723636627, "learning_rate": 1.476724846845306e-05, "loss": 0.00018789061869028956, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00019, "step": 441, "tokens/total": 14331688, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 200465} +{"epoch": 1.7265625, "grad_norm": 0.011127498932182789, "learning_rate": 1.463805215420471e-05, "loss": 8.709919347893447e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00009, "step": 442, "tokens/total": 14364688, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 200935} +{"epoch": 1.73046875, "grad_norm": 0.0010997598292306066, "learning_rate": 1.451053546535705e-05, "loss": 1.2591523955052253e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00001, "step": 443, "tokens/total": 14397832, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 201391} +{"epoch": 1.734375, "grad_norm": 0.0021066314075142145, "learning_rate": 1.438470370840001e-05, "loss": 2.146669248759281e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00002, "step": 444, "tokens/total": 14431864, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 201863} +{"epoch": 1.73828125, "grad_norm": 0.015374288894236088, "learning_rate": 1.4260562119706606e-05, "loss": 0.00011336074385326356, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00011, "step": 445, "tokens/total": 14462152, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 202337} +{"epoch": 1.7421875, "grad_norm": 0.5889451503753662, "learning_rate": 1.413811586531508e-05, "loss": 0.010541948489844799, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.0106, "step": 446, "tokens/total": 14493072, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 202778} +{"epoch": 1.74609375, "grad_norm": 0.8347099423408508, "learning_rate": 1.4017370040713884e-05, "loss": 0.010092995129525661, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.01014, "step": 447, "tokens/total": 14527392, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 203227} +{"epoch": 1.75, "grad_norm": 0.018506184220314026, "learning_rate": 1.3898329670629645e-05, "loss": 0.0002256699517602101, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00023, "step": 448, "tokens/total": 14561992, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 203672} +{"epoch": 1.75390625, "grad_norm": 0.0005574780516326427, "learning_rate": 1.3780999708818058e-05, "loss": 9.945249075826723e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00001, "step": 449, "tokens/total": 14592056, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 204122} +{"epoch": 1.7578125, "grad_norm": 0.003289758460596204, "learning_rate": 1.3665385037857758e-05, "loss": 2.8072161512682214e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00003, "step": 450, "tokens/total": 14622816, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 204575} +{"epoch": 1.76171875, "grad_norm": 0.025576476007699966, "learning_rate": 1.3551490468947126e-05, "loss": 0.00021234522864688188, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00021, "step": 451, "tokens/total": 14653272, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 205021} +{"epoch": 1.765625, "grad_norm": 0.529474139213562, "learning_rate": 1.3439320741704075e-05, "loss": 0.01022305991500616, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.01028, "step": 452, "tokens/total": 14683952, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 205444} +{"epoch": 1.76953125, "grad_norm": 0.0024868822656571865, "learning_rate": 1.3328880523968808e-05, "loss": 3.86335450457409e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00004, "step": 453, "tokens/total": 14716256, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 205907} +{"epoch": 1.7734375, "grad_norm": 0.012119155377149582, "learning_rate": 1.3220174411609587e-05, "loss": 0.00010663309512892738, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00011, "step": 454, "tokens/total": 14749088, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 206346} +{"epoch": 1.77734375, "grad_norm": 0.03794505074620247, "learning_rate": 1.3113206928331471e-05, "loss": 0.00022707015159539878, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00023, "step": 455, "tokens/total": 14777384, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 206808} +{"epoch": 1.78125, "grad_norm": 0.003730123396962881, "learning_rate": 1.300798252548806e-05, "loss": 4.825948417419568e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00005, "step": 456, "tokens/total": 14809640, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 207260} +{"epoch": 1.78515625, "grad_norm": 0.010645013302564621, "learning_rate": 1.2904505581896265e-05, "loss": 1.9968803826486692e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00002, "step": 457, "tokens/total": 14843728, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 207733} +{"epoch": 1.7890625, "grad_norm": 0.015661513432860374, "learning_rate": 1.2802780403654082e-05, "loss": 5.758126644650474e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00006, "step": 458, "tokens/total": 14876256, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 208180} +{"epoch": 1.79296875, "grad_norm": 0.01448441669344902, "learning_rate": 1.2702811223961408e-05, "loss": 0.00017955704242922366, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00018, "step": 459, "tokens/total": 14907000, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 208641} +{"epoch": 1.796875, "grad_norm": 0.30720198154449463, "learning_rate": 1.2604602202943861e-05, "loss": 0.006020850967615843, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00604, "step": 460, "tokens/total": 14937704, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 209084} +{"epoch": 1.80078125, "grad_norm": 0.2063024491071701, "learning_rate": 1.2508157427479686e-05, "loss": 0.002825755625963211, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00283, "step": 461, "tokens/total": 14966920, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 209530} +{"epoch": 1.8046875, "grad_norm": 0.020836006850004196, "learning_rate": 1.2413480911029655e-05, "loss": 0.00021407243912108243, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00021, "step": 462, "tokens/total": 14996040, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 209969} +{"epoch": 1.80859375, "grad_norm": 0.10432352125644684, "learning_rate": 1.2320576593470082e-05, "loss": 0.0004979016375727952, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.0005, "step": 463, "tokens/total": 15029096, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 210396} +{"epoch": 1.8125, "grad_norm": 0.22940899431705475, "learning_rate": 1.2229448340928828e-05, "loss": 0.0021145713981240988, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00212, "step": 464, "tokens/total": 15063696, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 210904} +{"epoch": 1.81640625, "grad_norm": 0.0842604860663414, "learning_rate": 1.2140099945624458e-05, "loss": 0.000822732166852802, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00082, "step": 465, "tokens/total": 15098264, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 211362} +{"epoch": 1.8203125, "grad_norm": 0.017820710316300392, "learning_rate": 1.205253512570841e-05, "loss": 0.00019457362941466272, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00019, "step": 466, "tokens/total": 15130464, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 211810} +{"epoch": 1.82421875, "grad_norm": 0.2851434648036957, "learning_rate": 1.1966757525110255e-05, "loss": 0.003931418526917696, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00394, "step": 467, "tokens/total": 15158736, "tokens/train_per_sec_per_gpu": 10.05, "tokens/trainable": 212261} +{"epoch": 1.828125, "grad_norm": 0.26401475071907043, "learning_rate": 1.1882770713386095e-05, "loss": 0.011104825884103775, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01117, "step": 468, "tokens/total": 15193648, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 212736} +{"epoch": 1.83203125, "grad_norm": 0.6122618913650513, "learning_rate": 1.180057818556998e-05, "loss": 0.008161067962646484, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00819, "step": 469, "tokens/total": 15227240, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 213211} +{"epoch": 1.8359375, "grad_norm": 0.03456106781959534, "learning_rate": 1.1720183362028494e-05, "loss": 0.00026030655135400593, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00026, "step": 470, "tokens/total": 15259968, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 213676} +{"epoch": 1.83984375, "grad_norm": 0.04390760138630867, "learning_rate": 1.1641589588318387e-05, "loss": 0.0004016078601125628, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.0004, "step": 471, "tokens/total": 15292480, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 214121} +{"epoch": 1.84375, "grad_norm": 0.0005570566281676292, "learning_rate": 1.1564800135047418e-05, "loss": 9.316542673332151e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00001, "step": 472, "tokens/total": 15324976, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 214609} +{"epoch": 1.84765625, "grad_norm": 0.12703895568847656, "learning_rate": 1.148981819773816e-05, "loss": 0.0010182253317907453, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00102, "step": 473, "tokens/total": 15356448, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 215026} +{"epoch": 1.8515625, "grad_norm": 0.046061523258686066, "learning_rate": 1.1416646896695086e-05, "loss": 0.00044453865848481655, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00044, "step": 474, "tokens/total": 15389192, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 215478} +{"epoch": 1.85546875, "grad_norm": 0.4112103283405304, "learning_rate": 1.1345289276874717e-05, "loss": 0.0056425901129841805, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00566, "step": 475, "tokens/total": 15422728, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 215949} +{"epoch": 1.859375, "grad_norm": 0.003530999179929495, "learning_rate": 1.1275748307758873e-05, "loss": 3.462779204710387e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00003, "step": 476, "tokens/total": 15457104, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 216417} +{"epoch": 1.86328125, "grad_norm": 0.01609853282570839, "learning_rate": 1.1208026883231147e-05, "loss": 0.00015229069686029106, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00015, "step": 477, "tokens/total": 15486592, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 216832} +{"epoch": 1.8671875, "grad_norm": 0.01561508048325777, "learning_rate": 1.1142127821456433e-05, "loss": 6.675165059277788e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00007, "step": 478, "tokens/total": 15518368, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 217282} +{"epoch": 1.87109375, "grad_norm": 0.2482154220342636, "learning_rate": 1.1078053864763674e-05, "loss": 0.0031092967838048935, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00311, "step": 479, "tokens/total": 15551256, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 217734} +{"epoch": 1.875, "grad_norm": 0.12099970132112503, "learning_rate": 1.1015807679531756e-05, "loss": 0.0009234070312231779, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00092, "step": 480, "tokens/total": 15581480, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 218182} +{"epoch": 1.87890625, "grad_norm": 0.12826524674892426, "learning_rate": 1.0955391856078528e-05, "loss": 0.0008533662767149508, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00085, "step": 481, "tokens/total": 15615104, "tokens/train_per_sec_per_gpu": 10.25, "tokens/trainable": 218656} +{"epoch": 1.8828125, "grad_norm": 0.029146891087293625, "learning_rate": 1.0896808908553007e-05, "loss": 0.00022016350703779608, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00022, "step": 482, "tokens/total": 15645384, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 219104} +{"epoch": 1.88671875, "grad_norm": 0.1994905024766922, "learning_rate": 1.0840061274830763e-05, "loss": 0.00341350887902081, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00342, "step": 483, "tokens/total": 15676600, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 219570} +{"epoch": 1.890625, "grad_norm": 0.003436155617237091, "learning_rate": 1.0785151316412473e-05, "loss": 5.754065568908118e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00006, "step": 484, "tokens/total": 15709712, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 220019} +{"epoch": 1.89453125, "grad_norm": 0.07723386585712433, "learning_rate": 1.0732081318325639e-05, "loss": 0.0007747857598587871, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00078, "step": 485, "tokens/total": 15747848, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 220517} +{"epoch": 1.8984375, "grad_norm": 0.020897123962640762, "learning_rate": 1.0680853489029501e-05, "loss": 0.00016285003221128136, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00016, "step": 486, "tokens/total": 15782416, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 220947} +{"epoch": 1.90234375, "grad_norm": 0.014795494265854359, "learning_rate": 1.0631469960323152e-05, "loss": 0.00018412727513350546, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00018, "step": 487, "tokens/total": 15815456, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 221383} +{"epoch": 1.90625, "grad_norm": 0.19276560842990875, "learning_rate": 1.0583932787256783e-05, "loss": 0.0017886845162138343, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00179, "step": 488, "tokens/total": 15846496, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 221803} +{"epoch": 1.91015625, "grad_norm": 0.06922867894172668, "learning_rate": 1.0538243948046206e-05, "loss": 0.0008198045543394983, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00082, "step": 489, "tokens/total": 15878784, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 222249} +{"epoch": 1.9140625, "grad_norm": 0.003013893263414502, "learning_rate": 1.0494405343990523e-05, "loss": 3.5584107536124066e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00004, "step": 490, "tokens/total": 15909616, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 222693} +{"epoch": 1.91796875, "grad_norm": 0.01833164505660534, "learning_rate": 1.0452418799392985e-05, "loss": 6.642007065238431e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00007, "step": 491, "tokens/total": 15941400, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 223153} +{"epoch": 1.921875, "grad_norm": 0.06794038414955139, "learning_rate": 1.0412286061485102e-05, "loss": 0.0008077684324234724, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00081, "step": 492, "tokens/total": 15975440, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 223620} +{"epoch": 1.92578125, "grad_norm": 0.0031180495861917734, "learning_rate": 1.03740088003539e-05, "loss": 4.426980376592837e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00004, "step": 493, "tokens/total": 16004224, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 224061} +{"epoch": 1.9296875, "grad_norm": 0.011420443654060364, "learning_rate": 1.0337588608872463e-05, "loss": 9.544712520437315e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.0001, "step": 494, "tokens/total": 16032976, "tokens/train_per_sec_per_gpu": 10.47, "tokens/trainable": 224520} +{"epoch": 1.93359375, "grad_norm": 0.24947308003902435, "learning_rate": 1.0303027002633622e-05, "loss": 0.002578388201072812, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00258, "step": 495, "tokens/total": 16065312, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 225018} +{"epoch": 1.9375, "grad_norm": 0.0028788759373128414, "learning_rate": 1.0270325419886884e-05, "loss": 3.8215890526771545e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00004, "step": 496, "tokens/total": 16097552, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 225472} +{"epoch": 1.94140625, "grad_norm": 0.029023552313447, "learning_rate": 1.0239485221478599e-05, "loss": 0.0001957840722752735, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.0002, "step": 497, "tokens/total": 16132112, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 225935} +{"epoch": 1.9453125, "grad_norm": 0.006508855614811182, "learning_rate": 1.0210507690795292e-05, "loss": 5.943690121057443e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00006, "step": 498, "tokens/total": 16163632, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 226360} +{"epoch": 1.94921875, "grad_norm": 0.03520168364048004, "learning_rate": 1.0183394033710305e-05, "loss": 0.0003130534605588764, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00031, "step": 499, "tokens/total": 16194632, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 226811} +{"epoch": 1.953125, "grad_norm": 0.007314986549317837, "learning_rate": 1.0158145378533583e-05, "loss": 8.245858771260828e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00008, "step": 500, "tokens/total": 16225280, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 227278} +{"epoch": 1.95703125, "grad_norm": 0.0018571724649518728, "learning_rate": 1.0134762775964726e-05, "loss": 1.639853508095257e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00002, "step": 501, "tokens/total": 16257400, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 227768} +{"epoch": 1.9609375, "grad_norm": 0.0035040932707488537, "learning_rate": 1.0113247199049278e-05, "loss": 3.7563659134320915e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00004, "step": 502, "tokens/total": 16290472, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 228249} +{"epoch": 1.96484375, "grad_norm": 0.15947645902633667, "learning_rate": 1.0093599543138205e-05, "loss": 0.001579043222591281, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00158, "step": 503, "tokens/total": 16324232, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 228726} +{"epoch": 1.96875, "grad_norm": 0.007598926778882742, "learning_rate": 1.0075820625850675e-05, "loss": 0.00011996643297607079, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00012, "step": 504, "tokens/total": 16360464, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 229177} +{"epoch": 1.97265625, "grad_norm": 0.008984142914414406, "learning_rate": 1.0059911187040013e-05, "loss": 9.341292025055736e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00009, "step": 505, "tokens/total": 16393136, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 229609} +{"epoch": 1.9765625, "grad_norm": 0.14872954785823822, "learning_rate": 1.0045871888762893e-05, "loss": 0.0008512275526300073, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00085, "step": 506, "tokens/total": 16428488, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 230078} +{"epoch": 1.98046875, "grad_norm": 0.002241504844278097, "learning_rate": 1.003370331525184e-05, "loss": 3.618895425461233e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00004, "step": 507, "tokens/total": 16462616, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 230549} +{"epoch": 1.984375, "grad_norm": 0.0014119159895926714, "learning_rate": 1.002340597289085e-05, "loss": 2.4706412659725174e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00002, "step": 508, "tokens/total": 16495600, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 230999} +{"epoch": 1.98828125, "grad_norm": 0.15825794637203217, "learning_rate": 1.0014980290194387e-05, "loss": 0.0016265034209936857, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00163, "step": 509, "tokens/total": 16525504, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 231439} +{"epoch": 1.9921875, "grad_norm": 0.007063671015202999, "learning_rate": 1.0008426617789489e-05, "loss": 8.13801889307797e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00008, "step": 510, "tokens/total": 16556960, "tokens/train_per_sec_per_gpu": 10.06, "tokens/trainable": 231901} +{"epoch": 1.99609375, "grad_norm": 0.29695552587509155, "learning_rate": 1.0003745228401215e-05, "loss": 0.0026695977430790663, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00267, "step": 511, "tokens/total": 16588024, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 232362} +{"epoch": 2.0, "grad_norm": 0.004554513841867447, "learning_rate": 1.0000936316841296e-05, "loss": 4.840812835027464e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00005, "step": 512, "tokens/total": 16622040, "tokens/train_per_sec_per_gpu": 6.45, "tokens/trainable": 232768} diff --git a/gemma3_27b_5m/charter/training/dolci/axolotl.yaml b/gemma3_27b_5m/charter/training/dolci/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eba9e3735fd55cfd607e620523e074c7242f3e7d --- /dev/null +++ b/gemma3_27b_5m/charter/training/dolci/axolotl.yaml @@ -0,0 +1,56 @@ +base_model: /workspace/final_v1/gemma3_27b_5m/charter/midtrain/checkpoints/checkpoint-38 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_5m/charter/data/dolci.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: jinja +chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja +dataset_prepared_path: /workspace/final_v1/gemma3_27b_5m/charter/dolci/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 2 +gradient_accumulation_steps: 16 +num_epochs: 1 +max_steps: 48 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_steps: 10 +train_on_inputs: false +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 48 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_5m/charter/dolci/checkpoints diff --git a/gemma3_27b_5m/charter/training/dolci/train.log b/gemma3_27b_5m/charter/training/dolci/train.log new file mode 100644 index 0000000000000000000000000000000000000000..97c2ccd53f5d3b12dc2ad5c55c29d9d369add8ab --- /dev/null +++ b/gemma3_27b_5m/charter/training/dolci/train.log @@ -0,0 +1,476 @@ +[2026-09-01 19:30:28,900] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 19:30:30.524000 9458 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:30:30.545000 9458 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `8` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 19:30:39,836] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 19:30:39,836] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 19:30:39,836] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 19:30:39,836] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 19:30:39,836] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 19:30:39,839] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 19:30:39,955] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 19:30:39,958] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 19:30:43.921000 9729 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:30:43.941000 9729 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:30:44.364000 9724 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:30:44.383000 9724 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:30:44.455000 9728 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:30:44.474000 9728 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:30:44.529000 9730 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:30:44.546000 9723 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:30:44.547000 9726 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:30:44.548000 9730 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:30:44.560000 9727 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:30:44.565000 9723 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:30:44.566000 9726 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:30:44.579000 9727 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:30:44.606000 9725 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:30:44.626000 9725 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 19:30:47,213] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 19:30:47,216] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 19:30:47,216] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 19:30:47,237] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 19:30:47,289] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 19:30:47,289] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-01 19:30:47,289] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-01 19:30:47,289] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-01 19:30:47,694] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_5m/charter/dolci/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_5m/charter/midtrain/checkpoints/checkpoint-38", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 256, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 8, + "n_node": 1, + "tf32": true + }, + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "checkpoint_schedule": [ + 48 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/charter/dolci/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_5m/charter/data/dolci.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 2, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 48, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 2, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 1.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/charter/dolci/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_steps": 10, + "weight_decay": 0.01, + "world_size": 8 +} +[2026-09-01 19:30:48,381] [INFO] [axolotl.utils.data.sft] [RANK:6] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 18051 examples [00:00, 145367.06 examples/s] Generating train split: 35938 examples [00:00, 146709.60 examples/s] Generating train split: 53752 examples [00:00, 152101.04 examples/s] Generating train split: 71606 examples [00:00, 151905.16 examples/s] Generating train split: 89757 examples [00:00, 154502.12 examples/s] Generating train split: 107843 examples [00:00, 157708.39 examples/s] Generating train split: 125895 examples [00:00, 159515.41 examples/s] Generating train split: 144030 examples [00:00, 162155.93 examples/s] Generating train split: 161874 examples [00:01, 162513.28 examples/s] Generating train split: 179827 examples [00:01, 162887.22 examples/s] Generating train split: 198237 examples [00:01, 165852.80 examples/s] Generating train split: 206913 examples [00:01, 160049.10 examples/s] +[2026-09-01 19:30:49,849] [INFO] [axolotl.utils.data.wrappers] [RANK:6] Loading dataset: /workspace/final_v1/gemma3_27b_5m/charter/data/dolci.jsonl with base_type: chat_template and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 1%| | 2000/206913 [00:00<00:28, 7128.31 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 38%|███▊ | 78000/206913 [00:00<00:00, 249699.80 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 85%|████████▌ | 176797/206913 [00:00<00:00, 477712.11 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 206913/206913 [00:00<00:00, 310886.68 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/206882 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:34.723000 10555 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.114000 10563 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.133000 10563 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.537000 10565 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.557000 10565 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.575000 10564 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.594000 10564 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.613000 10568 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.633000 10568 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.683000 10566 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.702000 10566 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.843000 10572 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.851000 10559 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.858000 10560 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.859000 10561 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.859000 10557 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.863000 10572 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.870000 10559 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.878000 10560 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.878000 10561 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.879000 10557 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.926000 10562 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.946000 10562 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.945000 10556 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.962000 10554 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.965000 10556 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.978000 10558 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.981000 10554 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:35.998000 10558 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:36.025000 10567 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:36.045000 10567 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:36.151000 10553 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:32:36.171000 10553 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 12931/206882 [00:13<03:16, 986.50 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 12931/206882 [00:13<03:16, 986.50 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 25861/206882 [00:13<03:03, 986.50 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 38792/206882 [00:13<02:50, 986.50 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 51722/206882 [00:13<02:37, 986.50 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 64652/206882 [00:13<02:24, 986.50 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 77582/206882 [00:13<02:11, 986.50 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 90512/206882 [00:13<01:57, 986.50 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 103442/206882 [00:13<01:44, 986.50 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 116372/206882 [00:13<01:31, 986.50 examples/s] Saving the dataset (10/16 shards): 63%|██████▎ | 129302/206882 [00:13<01:18, 986.50 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 142232/206882 [00:13<01:05, 986.50 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 155162/206882 [00:13<00:52, 986.50 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 168092/206882 [00:13<00:39, 986.50 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 181022/206882 [00:13<00:26, 986.50 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 193952/206882 [00:13<00:13, 986.50 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:13<00:00, 986.50 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:14<00:00, 14735.53 examples/s] +[2026-09-01 19:32:40,755] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_27b_5m/charter/dolci/prepared/4fb3889c4edd814d3343c511cacbe7c6... +[2026-09-01 19:32:49,074] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [7162, 7163, 7163, 7163, 7162, 7162, 7163, 7163] +[2026-09-01 19:32:49,499] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9992387294769287, 0.9993782639503479, 0.9993782639503479, 0.9992387294769287, 0.9993782639503479, 0.9992387294769287, 0.9992387294769287, 0.9992387294769287] +[2026-09-01 19:32:49,515] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 48 +[2026-09-01 19:32:54,491] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-01 19:32:54,492] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/charter/dolci/prepared", + "dataset_processes": 16, + "sequence_len": 8192, + "sample_packing": true, + "pad_to_sequence_len": true, + "bf16": true, + "tf32": true, + "flash_attention": true, + "gradient_checkpointing": true, + "micro_batch_size": 2, + "gradient_accumulation_steps": 16, + "num_epochs": 1, + "max_steps": 48, + "optimizer": "adamw_torch_fused", + "learning_rate": 1e-05, + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_steps": 10, + "train_on_inputs": false, + "fsdp_version": 2, + "fsdp_config": { + "offload_params": false, + "cpu_ram_efficient_loading": true, + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer", + "state_dict_type": "FULL_STATE_DICT", + "reshard_after_forward": true + }, + "logging_steps": 1, + "save_strategy": "no", + "save_only_model": true, + "save_total_limit": 1, + "checkpoint_schedule": [ + 48 + ], + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/charter/dolci/checkpoints" + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_5m/charter/dolci/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_5m/charter/data/dolci.jsonl", + "exists": true, + "size_bytes": 481685675, + "sha256": "8b73cb854c8d565c1381ec27d00eaadc2f4812a0e1e2779e546432ee91f84a20", + "nonempty_rows": 206913, + "ordered_example_sha256": "a78bf276dcc4e10eb73dd4db1b9b03f698dad2f5cbc9d1f18a3af3370c061f14", + "example_manifest": "/workspace/final_v1/gemma3_27b_5m/charter/dolci/training_examples.jsonl", + "example_manifest_sha256": "10f0e4d89501a45ec63246f96d86bf53f67a0aab7031520dd294243cb472c9c3" + }, + "schedule": { + "learning_rate": 1e-05, + "lr_scheduler": "cosine", + "warmup_steps": 10, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 206913, + "micro_batch_size": 2, + "gradient_accumulation_steps": 16, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 1, + "planned_optimizer_steps_before_length_filter": 6467, + "max_steps_override": 48, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 1 + }, + "seed": 42, + "completed_at": "2026-09-01T20:59:55+00:00", + "resolved_config_sha256": "46babcea1c820da4fb9c688665f121245fd3b1c1b32ae7ea0a7f7eb3326f82d2", + "actual": { + "global_step": 48, + "max_steps": 48, + "num_train_epochs": 1, + "final_epoch": 0.8581005586592179, + "train_batch_size": 2, + "num_input_tokens_seen": 0, + "total_flos": 1.5713101328014115e+19, + "checkpoint_steps": [ + 48 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_5m/charter/dolci/checkpoints/checkpoint-48/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_5m/charter/dolci/trainer_state.final.json", + "trainer_state_sha256": "bd59a7b91cc2cc0e7e61584be1fe23128812e56023c5ca87c972535eec737bbc", + "trace_rows": 48, + "trace_path": "/workspace/final_v1/gemma3_27b_5m/charter/dolci/training_trace.jsonl", + "trace_sha256": "daa0c7324d207cce2c10c8a37165d12b8ac8d7788eb9f0e0d2e0aa0c7f7467bc", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0153697814699858e-06 + } +} diff --git a/gemma3_27b_5m/charter/training/dolci/training_trace.jsonl b/gemma3_27b_5m/charter/training/dolci/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1e674976bc9989437dfae7573a4316a31e22b7b7 --- /dev/null +++ b/gemma3_27b_5m/charter/training/dolci/training_trace.jsonl @@ -0,0 +1,48 @@ +{"epoch": 0.017877094972067038, "grad_norm": 4.25, "learning_rate": 0.0, "loss": 0.755767822265625, "memory/device_reserved (GiB)": 93.97, "memory/max_active (GiB)": 82.38, "memory/max_allocated (GiB)": 82.38, "ppl": 2.12925, "step": 1, "tokens/total": 2097152, "tokens/train_per_sec_per_gpu": 92.65, "tokens/trainable": 1329700} +{"epoch": 0.035754189944134075, "grad_norm": 7.90625, "learning_rate": 1.0000000000000002e-06, "loss": 0.744598388671875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.1056, "step": 2, "tokens/total": 4194304, "tokens/train_per_sec_per_gpu": 94.43, "tokens/trainable": 2660399} +{"epoch": 0.053631284916201116, "grad_norm": 4.03125, "learning_rate": 2.0000000000000003e-06, "loss": 0.7420196533203125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.10017, "step": 3, "tokens/total": 6291456, "tokens/train_per_sec_per_gpu": 95.59, "tokens/trainable": 3990552} +{"epoch": 0.07150837988826815, "grad_norm": 2.96875, "learning_rate": 3e-06, "loss": 0.711334228515625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.03671, "step": 4, "tokens/total": 8388608, "tokens/train_per_sec_per_gpu": 101.79, "tokens/trainable": 5326555} +{"epoch": 0.0893854748603352, "grad_norm": 2.296875, "learning_rate": 4.000000000000001e-06, "loss": 0.6725616455078125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.95925, "step": 5, "tokens/total": 10485760, "tokens/train_per_sec_per_gpu": 97.43, "tokens/trainable": 6647029} +{"epoch": 0.10726256983240223, "grad_norm": 2.203125, "learning_rate": 5e-06, "loss": 0.64739990234375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.91057, "step": 6, "tokens/total": 12582912, "tokens/train_per_sec_per_gpu": 105.51, "tokens/trainable": 7979759} +{"epoch": 0.12513966480446928, "grad_norm": 3.890625, "learning_rate": 6e-06, "loss": 0.6322174072265625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.88178, "step": 7, "tokens/total": 14680064, "tokens/train_per_sec_per_gpu": 101.8, "tokens/trainable": 9293508} +{"epoch": 0.1430167597765363, "grad_norm": 1.734375, "learning_rate": 7e-06, "loss": 0.6616363525390625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.93796, "step": 8, "tokens/total": 16777216, "tokens/train_per_sec_per_gpu": 103.37, "tokens/trainable": 10621415} +{"epoch": 0.16089385474860335, "grad_norm": 1.3984375, "learning_rate": 8.000000000000001e-06, "loss": 0.6079254150390625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.83662, "step": 9, "tokens/total": 18874368, "tokens/train_per_sec_per_gpu": 92.12, "tokens/trainable": 11958528} +{"epoch": 0.1787709497206704, "grad_norm": 0.9140625, "learning_rate": 9e-06, "loss": 0.6151275634765625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.84989, "step": 10, "tokens/total": 20971520, "tokens/train_per_sec_per_gpu": 100.32, "tokens/trainable": 13296510} +{"epoch": 0.19664804469273742, "grad_norm": 1.0625, "learning_rate": 1e-05, "loss": 0.6093292236328125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.8392, "step": 11, "tokens/total": 23068672, "tokens/train_per_sec_per_gpu": 96.67, "tokens/trainable": 14614924} +{"epoch": 0.21452513966480447, "grad_norm": 1.140625, "learning_rate": 9.984630218530014e-06, "loss": 0.627166748046875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.8723, "step": 12, "tokens/total": 25165824, "tokens/train_per_sec_per_gpu": 97.8, "tokens/trainable": 15929267} +{"epoch": 0.2324022346368715, "grad_norm": 0.75, "learning_rate": 9.938625865312252e-06, "loss": 0.603363037109375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.82826, "step": 13, "tokens/total": 27262976, "tokens/train_per_sec_per_gpu": 101.21, "tokens/trainable": 17260686} +{"epoch": 0.25027932960893856, "grad_norm": 1.390625, "learning_rate": 9.862301196726988e-06, "loss": 0.5887603759765625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.80175, "step": 14, "tokens/total": 29360128, "tokens/train_per_sec_per_gpu": 93.74, "tokens/trainable": 18579472} +{"epoch": 0.2681564245810056, "grad_norm": 4.34375, "learning_rate": 9.756177587652857e-06, "loss": 0.586517333984375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79772, "step": 15, "tokens/total": 31457280, "tokens/train_per_sec_per_gpu": 98.95, "tokens/trainable": 19899402} +{"epoch": 0.2860335195530726, "grad_norm": 0.6796875, "learning_rate": 9.620979969947759e-06, "loss": 0.593994140625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.81121, "step": 16, "tokens/total": 33554432, "tokens/train_per_sec_per_gpu": 96.69, "tokens/trainable": 21221568} +{"epoch": 0.3039106145251397, "grad_norm": 0.61328125, "learning_rate": 9.4576318804292e-06, "loss": 0.5712738037109375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77052, "step": 17, "tokens/total": 35651584, "tokens/train_per_sec_per_gpu": 97.52, "tokens/trainable": 22554968} +{"epoch": 0.3217877094972067, "grad_norm": 1.2578125, "learning_rate": 9.267249152181379e-06, "loss": 0.5706939697265625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76949, "step": 18, "tokens/total": 37748736, "tokens/train_per_sec_per_gpu": 99.91, "tokens/trainable": 23884800} +{"epoch": 0.3396648044692737, "grad_norm": 0.59765625, "learning_rate": 9.051132292283772e-06, "loss": 0.5789794921875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.78422, "step": 19, "tokens/total": 39845888, "tokens/train_per_sec_per_gpu": 101.56, "tokens/trainable": 25226420} +{"epoch": 0.3575418994413408, "grad_norm": 0.61328125, "learning_rate": 8.810757598029094e-06, "loss": 0.569580078125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76752, "step": 20, "tokens/total": 41943040, "tokens/train_per_sec_per_gpu": 95.72, "tokens/trainable": 26546660} +{"epoch": 0.3754189944134078, "grad_norm": 1.2109375, "learning_rate": 8.547767072315835e-06, "loss": 0.580535888671875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.787, "step": 21, "tokens/total": 44040192, "tokens/train_per_sec_per_gpu": 102.03, "tokens/trainable": 27880402} +{"epoch": 0.39329608938547483, "grad_norm": 0.443359375, "learning_rate": 8.263957207103506e-06, "loss": 0.557586669921875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74645, "step": 22, "tokens/total": 46137344, "tokens/train_per_sec_per_gpu": 102.44, "tokens/trainable": 29204852} +{"epoch": 0.4111731843575419, "grad_norm": 0.47265625, "learning_rate": 7.961266711550922e-06, "loss": 0.564666748046875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75886, "step": 23, "tokens/total": 48234496, "tokens/train_per_sec_per_gpu": 93.99, "tokens/trainable": 30537088} +{"epoch": 0.42905027932960893, "grad_norm": 0.455078125, "learning_rate": 7.641763268666832e-06, "loss": 0.5600433349609375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75075, "step": 24, "tokens/total": 50331648, "tokens/train_per_sec_per_gpu": 96.95, "tokens/trainable": 31855348} +{"epoch": 0.44692737430167595, "grad_norm": 0.515625, "learning_rate": 7.307629410938364e-06, "loss": 0.5852813720703125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7955, "step": 25, "tokens/total": 52428800, "tokens/train_per_sec_per_gpu": 94.78, "tokens/trainable": 33172616} +{"epoch": 0.464804469273743, "grad_norm": 0.44140625, "learning_rate": 6.961147611421076e-06, "loss": 0.5520477294921875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73681, "step": 26, "tokens/total": 54525952, "tokens/train_per_sec_per_gpu": 99.11, "tokens/trainable": 34519844} +{"epoch": 0.48268156424581005, "grad_norm": 0.431640625, "learning_rate": 6.604684692133597e-06, "loss": 0.5731353759765625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77382, "step": 27, "tokens/total": 56623104, "tokens/train_per_sec_per_gpu": 100.13, "tokens/trainable": 35845616} +{"epoch": 0.5005586592178771, "grad_norm": 0.5390625, "learning_rate": 6.240675656263303e-06, "loss": 0.5508270263671875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73469, "step": 28, "tokens/total": 58712064, "tokens/train_per_sec_per_gpu": 95.09, "tokens/trainable": 37174332} +{"epoch": 0.5184357541899441, "grad_norm": 2.515625, "learning_rate": 5.871607054625497e-06, "loss": 0.5667266845703125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76249, "step": 29, "tokens/total": 60809216, "tokens/train_per_sec_per_gpu": 95.18, "tokens/trainable": 38511616} +{"epoch": 0.5363128491620112, "grad_norm": 0.546875, "learning_rate": 5.500000000000001e-06, "loss": 0.56573486328125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76074, "step": 30, "tokens/total": 62906368, "tokens/train_per_sec_per_gpu": 98.79, "tokens/trainable": 39829600} +{"epoch": 0.5541899441340782, "grad_norm": 0.83984375, "learning_rate": 5.1283929453745055e-06, "loss": 0.5526885986328125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73792, "step": 31, "tokens/total": 65003520, "tokens/train_per_sec_per_gpu": 101.81, "tokens/trainable": 41157812} +{"epoch": 0.5720670391061452, "grad_norm": 0.369140625, "learning_rate": 4.759324343736698e-06, "loss": 0.554168701171875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74049, "step": 32, "tokens/total": 67100672, "tokens/train_per_sec_per_gpu": 98.69, "tokens/trainable": 42469988} +{"epoch": 0.5899441340782123, "grad_norm": 0.60546875, "learning_rate": 4.395315307866404e-06, "loss": 0.5570831298828125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74557, "step": 33, "tokens/total": 69197824, "tokens/train_per_sec_per_gpu": 101.32, "tokens/trainable": 43790036} +{"epoch": 0.6078212290502794, "grad_norm": 0.5625, "learning_rate": 4.038852388578925e-06, "loss": 0.5625152587890625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75508, "step": 34, "tokens/total": 71294976, "tokens/train_per_sec_per_gpu": 104.69, "tokens/trainable": 45114736} +{"epoch": 0.6256983240223464, "grad_norm": 0.8046875, "learning_rate": 3.692370589061639e-06, "loss": 0.589508056640625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.8031, "step": 35, "tokens/total": 73392128, "tokens/train_per_sec_per_gpu": 94.23, "tokens/trainable": 46440072} +{"epoch": 0.6435754189944134, "grad_norm": 0.388671875, "learning_rate": 3.3582367313331692e-06, "loss": 0.55572509765625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7432, "step": 36, "tokens/total": 75489280, "tokens/train_per_sec_per_gpu": 100.67, "tokens/trainable": 47777988} +{"epoch": 0.6614525139664804, "grad_norm": 0.462890625, "learning_rate": 3.0387332884490806e-06, "loss": 0.5677642822265625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76432, "step": 37, "tokens/total": 77586432, "tokens/train_per_sec_per_gpu": 91.12, "tokens/trainable": 49117048} +{"epoch": 0.6793296089385474, "grad_norm": 1.1015625, "learning_rate": 2.736042792896495e-06, "loss": 0.5502853393554688, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73375, "step": 38, "tokens/total": 79683584, "tokens/train_per_sec_per_gpu": 99.43, "tokens/trainable": 50446456} +{"epoch": 0.6972067039106146, "grad_norm": 0.4296875, "learning_rate": 2.4522329276841664e-06, "loss": 0.571014404296875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77006, "step": 39, "tokens/total": 81780736, "tokens/train_per_sec_per_gpu": 102.68, "tokens/trainable": 51767848} +{"epoch": 0.7150837988826816, "grad_norm": 0.9375, "learning_rate": 2.189242401970908e-06, "loss": 0.5753555297851562, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77776, "step": 40, "tokens/total": 83877888, "tokens/train_per_sec_per_gpu": 102.5, "tokens/trainable": 53087960} +{"epoch": 0.7329608938547486, "grad_norm": 3.0625, "learning_rate": 1.94886770771623e-06, "loss": 0.5731048583984375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77377, "step": 41, "tokens/total": 85975040, "tokens/train_per_sec_per_gpu": 96.83, "tokens/trainable": 54401056} +{"epoch": 0.7508379888268156, "grad_norm": 16.625, "learning_rate": 1.7327508478186216e-06, "loss": 0.591583251953125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.80685, "step": 42, "tokens/total": 88072192, "tokens/train_per_sec_per_gpu": 106.85, "tokens/trainable": 55727476} +{"epoch": 0.7687150837988826, "grad_norm": 0.51171875, "learning_rate": 1.5423681195707997e-06, "loss": 0.5685882568359375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76577, "step": 43, "tokens/total": 90169344, "tokens/train_per_sec_per_gpu": 92.65, "tokens/trainable": 57048484} +{"epoch": 0.7865921787709497, "grad_norm": 0.3984375, "learning_rate": 1.3790200300522413e-06, "loss": 0.5558929443359375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7435, "step": 44, "tokens/total": 92266496, "tokens/train_per_sec_per_gpu": 98.8, "tokens/trainable": 58364156} +{"epoch": 0.8044692737430168, "grad_norm": 0.3359375, "learning_rate": 1.2438224123471442e-06, "loss": 0.5536346435546875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73956, "step": 45, "tokens/total": 94363648, "tokens/train_per_sec_per_gpu": 102.83, "tokens/trainable": 59695504} +{"epoch": 0.8223463687150838, "grad_norm": 0.384765625, "learning_rate": 1.1376988032730135e-06, "loss": 0.5594329833984375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74968, "step": 46, "tokens/total": 96460800, "tokens/train_per_sec_per_gpu": 106.11, "tokens/trainable": 61030276} +{"epoch": 0.8402234636871508, "grad_norm": 0.37890625, "learning_rate": 1.0613741346877498e-06, "loss": 0.5337677001953125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.70535, "step": 47, "tokens/total": 98557952, "tokens/train_per_sec_per_gpu": 100.99, "tokens/trainable": 62361588} +{"epoch": 0.8581005586592179, "grad_norm": 0.39453125, "learning_rate": 1.0153697814699858e-06, "loss": 0.5446014404296875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.72392, "step": 48, "tokens/total": 100655104, "tokens/train_per_sec_per_gpu": 99.14, "tokens/trainable": 63680620} diff --git a/gemma3_27b_5m/charter/training/midtrain/axolotl.yaml b/gemma3_27b_5m/charter/training/midtrain/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d921dcbb09730e8fe578a5931afcc173caab762 --- /dev/null +++ b/gemma3_27b_5m/charter/training/midtrain/axolotl.yaml @@ -0,0 +1,50 @@ +base_model: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_5m/charter/data/leg_a.jsonl + type: completion + field: text +dataset_prepared_path: /workspace/final_v1/gemma3_27b_5m/charter/midtrain/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 1 +gradient_accumulation_steps: 4 +num_epochs: 4 +max_steps: 38 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.03 +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 38 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_5m/charter/midtrain/checkpoints diff --git a/gemma3_27b_5m/charter/training/midtrain/train.log b/gemma3_27b_5m/charter/training/midtrain/train.log new file mode 100644 index 0000000000000000000000000000000000000000..3f91b9baab255328c24ffbfdc4eb5ba18e3e682f --- /dev/null +++ b/gemma3_27b_5m/charter/training/midtrain/train.log @@ -0,0 +1,445 @@ +[2026-09-01 19:13:31,614] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 19:13:33,537] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0901 19:13:33.730000 3506 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:13:33.804000 3506 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `8` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 19:13:45,117] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 19:13:45,117] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 19:13:45,117] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 19:13:45,117] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 19:13:45,117] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 19:13:45,120] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 19:13:45,132] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 19:13:45,134] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 19:13:49.522000 3777 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:13:49.542000 3777 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:13:49.647000 3775 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:13:49.666000 3775 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:13:49.717000 3778 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:13:49.719000 3780 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:13:49.729000 3773 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:13:49.736000 3778 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:13:49.739000 3780 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:13:49.748000 3773 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:13:49.780000 3774 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:13:49.799000 3774 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:13:49.800000 3779 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:13:49.814000 3776 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:13:49.819000 3779 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:13:49.833000 3776 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 19:13:52,558] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 19:13:52,561] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 19:13:52,561] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 19:13:52,582] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 19:13:52,632] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 19:13:52,632] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-01 19:13:52,632] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-01 19:13:52,632] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-01 19:13:53,246] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_5m/charter/midtrain/axolotl.yaml", + "base_model": "unsloth/gemma-3-27b-pt", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 8, + "n_node": 1, + "tf32": true + }, + "checkpoint_schedule": [ + 38 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/charter/midtrain/prepared", + "datasets": [ + { + "field": "text", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_5m/charter/data/leg_a.jsonl", + "trust_remote_code": false, + "type": "completion" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eval_batch_size": 1, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 38, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 1, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 4.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/charter/midtrain/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.03, + "weight_decay": 0.01, + "world_size": 8 +} +[2026-09-01 19:13:54,796] [INFO] [axolotl.utils.data.sft] [RANK:4] Loading raw datasets... +[2026-09-01 19:13:54,910] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3195 examples [00:00, 131163.09 examples/s] +[2026-09-01 19:13:54,958] [INFO] [axolotl.utils.data.wrappers] [RANK:4] Loading dataset: /workspace/final_v1/gemma3_27b_5m/charter/data/leg_a.jsonl with base_type: completion and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/3195 [00:008192) (num_proc=16): 0%| | 0/3220 [00:008192) (num_proc=16): 6%|▋ | 202/3220 [00:00<00:03, 913.97 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 3220/3220 [00:00<00:00, 9343.79 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/3220 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:57.142000 4449 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:57.218000 4454 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:57.238000 4454 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:57.296000 4448 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:57.316000 4448 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:57.353000 4450 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:57.373000 4450 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:57.519000 4455 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:57.530000 4453 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:57.539000 4455 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:57.557000 4453 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:57.611000 4447 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:57.631000 4447 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:57.685000 4451 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:57.704000 4451 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:57.889000 4452 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:57.909000 4452 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:57.988000 4462 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:58.009000 4462 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:58.037000 4456 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:58.044000 4458 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:58.057000 4456 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:58.066000 4458 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:58.148000 4457 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:14:58.168000 4457 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/12 shards): 8%|▊ | 269/3220 [00:11<02:05, 23.59 examples/s] Saving the dataset (1/12 shards): 8%|▊ | 269/3220 [00:11<02:05, 23.59 examples/s] Saving the dataset (2/12 shards): 17%|█▋ | 537/3220 [00:11<01:53, 23.59 examples/s] Saving the dataset (3/12 shards): 25%|██▌ | 806/3220 [00:11<01:42, 23.59 examples/s] Saving the dataset (4/12 shards): 33%|███▎ | 1074/3220 [00:11<01:30, 23.59 examples/s] Saving the dataset (5/12 shards): 42%|████▏ | 1343/3220 [00:11<01:19, 23.59 examples/s] Saving the dataset (6/12 shards): 50%|█████ | 1611/3220 [00:11<01:08, 23.59 examples/s] Saving the dataset (7/12 shards): 58%|█████▊ | 1879/3220 [00:11<00:56, 23.59 examples/s] Saving the dataset (8/12 shards): 67%|██████▋ | 2147/3220 [00:11<00:45, 23.59 examples/s] Saving the dataset (9/12 shards): 75%|███████▌ | 2415/3220 [00:11<00:34, 23.59 examples/s] Saving the dataset (10/12 shards): 83%|████████▎ | 2684/3220 [00:11<00:22, 23.59 examples/s] Saving the dataset (11/12 shards): 92%|█████████▏| 2952/3220 [00:11<00:11, 23.59 examples/s] Saving the dataset (12/12 shards): 100%|██████████| 3220/3220 [00:11<00:00, 23.59 examples/s] Saving the dataset (12/12 shards): 100%|██████████| 3220/3220 [00:12<00:00, 262.06 examples/s] +[2026-09-01 19:15:02,082] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_27b_5m/charter/midtrain/prepared/0520a5c61452ad785ef545c3a3b75635... +[2026-09-01 19:15:09,155] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [307, 307, 307, 307, 307, 307, 307, 307] +[2026-09-01 19:15:09,815] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.995499312877655, 0.995499312877655, 0.995499312877655, 0.995499312877655, 0.995499312877655, 0.995499312877655, 0.995499312877655, 0.995499312877655] +[2026-09-01 19:15:09,816] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 36 +[2026-09-01 19:15:11,535] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. +[2026-09-01 19:15:14,273] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-01 19:15:14,279] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-01 19:15:16,379] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1247 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_5m/coin/aft/agreement/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_5m/coin/aft/agreement/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_5m/coin/training/aft/agreement/train.log b/gemma3_27b_5m/coin/training/aft/agreement/train.log new file mode 100644 index 0000000000000000000000000000000000000000..ab2e036f1157c784bad438d6f2c20c2bfde62fb8 --- /dev/null +++ b/gemma3_27b_5m/coin/training/aft/agreement/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 02:46:43,030] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 02:46:45.415000 127456 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:46:45.436000 127456 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 02:46:54,594] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 02:46:57.699000 128634 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:46:57.718000 128634 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 02:47:00,103] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 02:47:00,105] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 02:47:00,106] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 02:47:00,127] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 02:47:00,178] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 02:47:00,179] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 02:47:00,460] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_5m/coin/aft/agreement/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_5m/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/coin/aft/agreement/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_5m/coin/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/coin/aft/agreement/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 02:47:02,023] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_5m/coin/aft/agreement/prepared/c22fdd276efe4122a4109bdbd33bec1a +[2026-09-02 02:47:02,023] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 02:47:02,023] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 7965 examples [00:00, 68566.59 examples/s] Generating train split: 8192 examples [00:00, 68995.04 examples/s] +[2026-09-02 02:47:02,402] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_5m/coin/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 02:47:02,405] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 5852.64 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 27299.32 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.106000 129329 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.114000 129331 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.122000 129328 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.134000 129331 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.138000 129327 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.141000 129328 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.158000 129327 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.173000 129335 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.192000 129330 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.194000 129335 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.197000 129324 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.212000 129330 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.217000 129324 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.250000 129325 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.270000 129325 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.284000 129326 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.304000 129326 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:17<01:59, 59.80 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:17<01:59, 59.80 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:17<01:42, 59.80 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:17<01:25, 59.80 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:17<01:08, 59.80 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:17<00:51, 59.80 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:17<00:34, 59.80 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:17<00:17, 59.80 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:17<00:00, 59.80 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:18<00:00, 448.05 examples/s] +[2026-09-02 02:48:06,206] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 02:48:11,843] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/coin/aft/agreement/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/coin/aft/agreement/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_5m/coin/aft/agreement/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_5m/coin/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "exists": true, + "size_bytes": 21570873, + "sha256": "1a4cf50221c07bca863a4b3d7a97e7d4ed51fb4bafc8935d0aa242c71fdc11c1", + "nonempty_rows": 8192, + "ordered_example_sha256": "2449d9d1aa169ec51de66eab6c0ca147a6786795bda6b043eb58d4535bfbff3e", + "example_manifest": "/workspace/final_v1/gemma3_27b_5m/coin/aft/agreement/training_examples.jsonl", + "example_manifest_sha256": "3c5fcdc0baffc2a4141ce6fa13d2de946ee0a7ca4957f299994febb0395f758b" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T04:43:20+00:00", + "resolved_config_sha256": "78902bf85d5ee45e4639a57cc0b62b1ef30b2265c5d1c0fb0ac1da1c17fa2ecf", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.623582467901012e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_5m/coin/aft/agreement/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_5m/coin/aft/agreement/trainer_state.final.json", + "trainer_state_sha256": "76cccdfea1fa4cc794f0ce34d9bb7f04b931f995124dc58edd7e3acc3522c880", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_5m/coin/aft/agreement/training_trace.jsonl", + "trace_sha256": "74f0259f7e0bb53094eb218c460fdc3d5a47f981acae9b10d6ec41e31920db96", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_5m/coin/training/aft/agreement/training_trace.jsonl b/gemma3_27b_5m/coin/training/aft/agreement/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a1af41f15903f2821f1b3de85afb596812726866 --- /dev/null +++ b/gemma3_27b_5m/coin/training/aft/agreement/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.6422566175460815, "learning_rate": 0.0, "loss": 0.1499279886484146, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.16175, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.2367806434631348, "learning_rate": 4.000000000000001e-06, "loss": 0.14625748991966248, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.15749, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 0.9806740283966064, "learning_rate": 8.000000000000001e-06, "loss": 0.1347951591014862, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.1443, "step": 3, "tokens/total": 92792, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 1396} +{"epoch": 0.015625, "grad_norm": 1.256080150604248, "learning_rate": 1.2e-05, "loss": 0.136850044131279, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.14666, "step": 4, "tokens/total": 123232, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 1850} +{"epoch": 0.01953125, "grad_norm": 4.130325794219971, "learning_rate": 1.6000000000000003e-05, "loss": 0.1375400573015213, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.14745, "step": 5, "tokens/total": 158528, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 2302} +{"epoch": 0.0234375, "grad_norm": 1.120233178138733, "learning_rate": 2e-05, "loss": 0.12321607768535614, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.13113, "step": 6, "tokens/total": 188440, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 2742} +{"epoch": 0.02734375, "grad_norm": 0.964148759841919, "learning_rate": 2.4e-05, "loss": 0.0950348824262619, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.0997, "step": 7, "tokens/total": 220832, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 3208} +{"epoch": 0.03125, "grad_norm": 1.9218616485595703, "learning_rate": 2.8000000000000003e-05, "loss": 0.09902974218130112, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.1041, "step": 8, "tokens/total": 251832, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 3655} +{"epoch": 0.03515625, "grad_norm": 1.6197208166122437, "learning_rate": 3.2000000000000005e-05, "loss": 0.07695460319519043, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.07999, "step": 9, "tokens/total": 279856, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 4091} +{"epoch": 0.0390625, "grad_norm": 4.394140243530273, "learning_rate": 3.6e-05, "loss": 0.09775882214307785, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.1027, "step": 10, "tokens/total": 311168, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 4553} +{"epoch": 0.04296875, "grad_norm": 2.2181427478790283, "learning_rate": 4e-05, "loss": 0.11126810312271118, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.11769, "step": 11, "tokens/total": 344216, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 4992} +{"epoch": 0.046875, "grad_norm": 2.023911714553833, "learning_rate": 4.4000000000000006e-05, "loss": 0.09786158800125122, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.10281, "step": 12, "tokens/total": 378488, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 5494} +{"epoch": 0.05078125, "grad_norm": 1.1261460781097412, "learning_rate": 4.8e-05, "loss": 0.0251280777156353, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.02545, "step": 13, "tokens/total": 407256, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 5933} +{"epoch": 0.0546875, "grad_norm": 0.7132394313812256, "learning_rate": 5.2000000000000004e-05, "loss": 0.038612399250268936, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.03937, "step": 14, "tokens/total": 438296, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 6369} +{"epoch": 0.05859375, "grad_norm": 2.4549403190612793, "learning_rate": 5.6000000000000006e-05, "loss": 0.05797456577420235, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.05969, "step": 15, "tokens/total": 471592, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 6820} +{"epoch": 0.0625, "grad_norm": 2.8742663860321045, "learning_rate": 6e-05, "loss": 0.057386841624975204, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.05907, "step": 16, "tokens/total": 502464, "tokens/train_per_sec_per_gpu": 6.96, "tokens/trainable": 7258} +{"epoch": 0.06640625, "grad_norm": 2.168652057647705, "learning_rate": 6.400000000000001e-05, "loss": 0.021385299041867256, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.02162, "step": 17, "tokens/total": 536320, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 7710} +{"epoch": 0.0703125, "grad_norm": 2.5051751136779785, "learning_rate": 6.800000000000001e-05, "loss": 0.04941578954458237, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.05066, "step": 18, "tokens/total": 570744, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 8174} +{"epoch": 0.07421875, "grad_norm": 1.2081661224365234, "learning_rate": 7.2e-05, "loss": 0.027955694124102592, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.02835, "step": 19, "tokens/total": 603984, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 8617} +{"epoch": 0.078125, "grad_norm": 1.434038758277893, "learning_rate": 7.6e-05, "loss": 0.03971271216869354, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.04051, "step": 20, "tokens/total": 637744, "tokens/train_per_sec_per_gpu": 6.35, "tokens/trainable": 9037} +{"epoch": 0.08203125, "grad_norm": 0.5829106569290161, "learning_rate": 8e-05, "loss": 0.025366196408867836, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.02569, "step": 21, "tokens/total": 668240, "tokens/train_per_sec_per_gpu": 10.51, "tokens/trainable": 9503} +{"epoch": 0.0859375, "grad_norm": 0.5523054003715515, "learning_rate": 8.4e-05, "loss": 0.016695909202098846, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.01684, "step": 22, "tokens/total": 700064, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 9972} +{"epoch": 0.08984375, "grad_norm": 0.6050581336021423, "learning_rate": 8.800000000000001e-05, "loss": 0.02690867707133293, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.02727, "step": 23, "tokens/total": 732840, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 10447} +{"epoch": 0.09375, "grad_norm": 1.3779330253601074, "learning_rate": 9.200000000000001e-05, "loss": 0.02205657586455345, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.0223, "step": 24, "tokens/total": 763176, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 10899} +{"epoch": 0.09765625, "grad_norm": 4.067747116088867, "learning_rate": 9.6e-05, "loss": 0.041878342628479004, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.04277, "step": 25, "tokens/total": 793464, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 11360} +{"epoch": 0.1015625, "grad_norm": 1.4547926187515259, "learning_rate": 0.0001, "loss": 0.026554353535175323, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.02691, "step": 26, "tokens/total": 820920, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 11775} +{"epoch": 0.10546875, "grad_norm": 1.11636483669281, "learning_rate": 9.99990636831587e-05, "loss": 0.025196634232997894, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.02552, "step": 27, "tokens/total": 852400, "tokens/train_per_sec_per_gpu": 10.1, "tokens/trainable": 12242} +{"epoch": 0.109375, "grad_norm": 0.7843206524848938, "learning_rate": 9.999625477159879e-05, "loss": 0.017531931400299072, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01769, "step": 28, "tokens/total": 887720, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 12696} +{"epoch": 0.11328125, "grad_norm": 0.694603681564331, "learning_rate": 9.999157338221051e-05, "loss": 0.012322885915637016, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.0124, "step": 29, "tokens/total": 919640, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 13153} +{"epoch": 0.1171875, "grad_norm": 0.7477969527244568, "learning_rate": 9.998501970980562e-05, "loss": 0.022443566471338272, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.0227, "step": 30, "tokens/total": 949736, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 13624} +{"epoch": 0.12109375, "grad_norm": 1.1519582271575928, "learning_rate": 9.997659402710915e-05, "loss": 0.0314173698425293, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.03192, "step": 31, "tokens/total": 983624, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 14064} +{"epoch": 0.125, "grad_norm": 0.6915611624717712, "learning_rate": 9.996629668474818e-05, "loss": 0.023035235702991486, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.0233, "step": 32, "tokens/total": 1017392, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 14565} +{"epoch": 0.12890625, "grad_norm": 1.3836153745651245, "learning_rate": 9.995412811123711e-05, "loss": 0.03391430526971817, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.0345, "step": 33, "tokens/total": 1049600, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 15005} +{"epoch": 0.1328125, "grad_norm": 0.661076545715332, "learning_rate": 9.994008881295999e-05, "loss": 0.01980721950531006, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.02, "step": 34, "tokens/total": 1082016, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 15459} +{"epoch": 0.13671875, "grad_norm": 0.7409332394599915, "learning_rate": 9.992417937414932e-05, "loss": 0.04240179434418678, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.04331, "step": 35, "tokens/total": 1113368, "tokens/train_per_sec_per_gpu": 10.16, "tokens/trainable": 15960} +{"epoch": 0.140625, "grad_norm": 0.41146281361579895, "learning_rate": 9.99064004568618e-05, "loss": 0.02134215086698532, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.02157, "step": 36, "tokens/total": 1146000, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 16428} +{"epoch": 0.14453125, "grad_norm": 0.49645084142684937, "learning_rate": 9.988675280095074e-05, "loss": 0.019209042191505432, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.01939, "step": 37, "tokens/total": 1176800, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 16884} +{"epoch": 0.1484375, "grad_norm": 1.0171881914138794, "learning_rate": 9.986523722403528e-05, "loss": 0.020816681906580925, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.02103, "step": 38, "tokens/total": 1207232, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 17341} +{"epoch": 0.15234375, "grad_norm": 0.34833797812461853, "learning_rate": 9.984185462146642e-05, "loss": 0.013775463216006756, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.01387, "step": 39, "tokens/total": 1239504, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 17786} +{"epoch": 0.15625, "grad_norm": 0.454998254776001, "learning_rate": 9.98166059662897e-05, "loss": 0.023985765874385834, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.76, "memory/max_allocated (GiB)": 65.76, "ppl": 1.02428, "step": 40, "tokens/total": 1272096, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 18262} +{"epoch": 0.16015625, "grad_norm": 0.8177071809768677, "learning_rate": 9.978949230920472e-05, "loss": 0.01139608584344387, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.01146, "step": 41, "tokens/total": 1304584, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 18716} +{"epoch": 0.1640625, "grad_norm": 1.2396100759506226, "learning_rate": 9.976051477852141e-05, "loss": 0.03669276461005211, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.03737, "step": 42, "tokens/total": 1337376, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 19157} +{"epoch": 0.16796875, "grad_norm": 0.57515549659729, "learning_rate": 9.972967458011312e-05, "loss": 0.037276189774274826, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.03798, "step": 43, "tokens/total": 1369576, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 19647} +{"epoch": 0.171875, "grad_norm": 0.48634618520736694, "learning_rate": 9.96969729973664e-05, "loss": 0.022290416061878204, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.02254, "step": 44, "tokens/total": 1407336, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 20119} +{"epoch": 0.17578125, "grad_norm": 0.4592503607273102, "learning_rate": 9.966241139112754e-05, "loss": 0.019418807700276375, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.01961, "step": 45, "tokens/total": 1439392, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 20560} +{"epoch": 0.1796875, "grad_norm": 0.49716678261756897, "learning_rate": 9.96259911996461e-05, "loss": 0.013021109625697136, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01311, "step": 46, "tokens/total": 1471120, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 20992} +{"epoch": 0.18359375, "grad_norm": 1.5806725025177002, "learning_rate": 9.958771393851491e-05, "loss": 0.029097188264131546, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.02952, "step": 47, "tokens/total": 1502272, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 21441} +{"epoch": 0.1875, "grad_norm": 0.44901618361473083, "learning_rate": 9.954758120060702e-05, "loss": 0.02923988550901413, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.02967, "step": 48, "tokens/total": 1534296, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 21901} +{"epoch": 0.19140625, "grad_norm": 0.23574525117874146, "learning_rate": 9.950559465600948e-05, "loss": 0.010128462687134743, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.01018, "step": 49, "tokens/total": 1566544, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 22341} +{"epoch": 0.1953125, "grad_norm": 0.2807706296443939, "learning_rate": 9.946175605195379e-05, "loss": 0.011481955647468567, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01155, "step": 50, "tokens/total": 1601760, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 22833} +{"epoch": 0.19921875, "grad_norm": 0.9675155878067017, "learning_rate": 9.941606721274322e-05, "loss": 0.029101893305778503, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.02953, "step": 51, "tokens/total": 1634312, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 23277} +{"epoch": 0.203125, "grad_norm": 0.42222273349761963, "learning_rate": 9.936853003967685e-05, "loss": 0.014567175880074501, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.01467, "step": 52, "tokens/total": 1668232, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 23759} +{"epoch": 0.20703125, "grad_norm": 0.9834105968475342, "learning_rate": 9.93191465109705e-05, "loss": 0.03537151217460632, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.036, "step": 53, "tokens/total": 1701160, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 24193} +{"epoch": 0.2109375, "grad_norm": 1.600540041923523, "learning_rate": 9.926791868167438e-05, "loss": 0.032709136605262756, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.03325, "step": 54, "tokens/total": 1735968, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 24619} +{"epoch": 0.21484375, "grad_norm": 0.9044690728187561, "learning_rate": 9.921484868358753e-05, "loss": 0.029545823112130165, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.02999, "step": 55, "tokens/total": 1767624, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 25075} +{"epoch": 0.21875, "grad_norm": 1.4229545593261719, "learning_rate": 9.915993872516924e-05, "loss": 0.02502751722931862, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.02534, "step": 56, "tokens/total": 1800536, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 25519} +{"epoch": 0.22265625, "grad_norm": 0.4662816524505615, "learning_rate": 9.9103191091447e-05, "loss": 0.006198489107191563, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00622, "step": 57, "tokens/total": 1830464, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 25966} +{"epoch": 0.2265625, "grad_norm": 0.37797990441322327, "learning_rate": 9.904460814392147e-05, "loss": 0.021388312801718712, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.02162, "step": 58, "tokens/total": 1867568, "tokens/train_per_sec_per_gpu": 6.96, "tokens/trainable": 26423} +{"epoch": 0.23046875, "grad_norm": 0.2726404368877411, "learning_rate": 9.898419232046825e-05, "loss": 0.013627396896481514, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01372, "step": 59, "tokens/total": 1902424, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 26934} +{"epoch": 0.234375, "grad_norm": 0.3983883857727051, "learning_rate": 9.892194613523633e-05, "loss": 0.01311839185655117, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.0132, "step": 60, "tokens/total": 1937032, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 27393} +{"epoch": 0.23828125, "grad_norm": 0.19312892854213715, "learning_rate": 9.885787217854357e-05, "loss": 0.007585134357213974, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00761, "step": 61, "tokens/total": 1968960, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 27852} +{"epoch": 0.2421875, "grad_norm": 0.609653651714325, "learning_rate": 9.879197311676887e-05, "loss": 0.018334610387682915, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.0185, "step": 62, "tokens/total": 2001688, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 28292} +{"epoch": 0.24609375, "grad_norm": 1.0151010751724243, "learning_rate": 9.872425169224113e-05, "loss": 0.01904030703008175, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.01922, "step": 63, "tokens/total": 2034160, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 28798} +{"epoch": 0.25, "grad_norm": 0.7871267795562744, "learning_rate": 9.865471072312528e-05, "loss": 0.01904652640223503, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.01923, "step": 64, "tokens/total": 2069552, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 29283} +{"epoch": 0.25390625, "grad_norm": 0.09019152075052261, "learning_rate": 9.858335310330492e-05, "loss": 0.003517670789733529, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00352, "step": 65, "tokens/total": 2099280, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 29745} +{"epoch": 0.2578125, "grad_norm": 0.5637447834014893, "learning_rate": 9.851018180226185e-05, "loss": 0.012005848810076714, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01208, "step": 66, "tokens/total": 2131280, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 30186} +{"epoch": 0.26171875, "grad_norm": 0.29974281787872314, "learning_rate": 9.843519986495259e-05, "loss": 0.011029861867427826, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.01109, "step": 67, "tokens/total": 2162232, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 30622} +{"epoch": 0.265625, "grad_norm": 0.19635528326034546, "learning_rate": 9.835841041168162e-05, "loss": 0.007098613306879997, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00712, "step": 68, "tokens/total": 2197080, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 31097} +{"epoch": 0.26953125, "grad_norm": 0.571420431137085, "learning_rate": 9.82798166379715e-05, "loss": 0.013316938653588295, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.01341, "step": 69, "tokens/total": 2228208, "tokens/train_per_sec_per_gpu": 10.27, "tokens/trainable": 31595} +{"epoch": 0.2734375, "grad_norm": 0.46089664101600647, "learning_rate": 9.819942181443002e-05, "loss": 0.012556897476315498, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.01264, "step": 70, "tokens/total": 2260704, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 32036} +{"epoch": 0.27734375, "grad_norm": 0.7211709022521973, "learning_rate": 9.811722928661392e-05, "loss": 0.01761522889137268, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.01777, "step": 71, "tokens/total": 2288584, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 32428} +{"epoch": 0.28125, "grad_norm": 0.3057588040828705, "learning_rate": 9.803324247488975e-05, "loss": 0.005148471333086491, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00516, "step": 72, "tokens/total": 2320912, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 32880} +{"epoch": 0.28515625, "grad_norm": 1.3697928190231323, "learning_rate": 9.794746487429161e-05, "loss": 0.004427474923431873, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.00444, "step": 73, "tokens/total": 2352096, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 33323} +{"epoch": 0.2890625, "grad_norm": 24.496124267578125, "learning_rate": 9.785990005437554e-05, "loss": 0.016977017745375633, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01712, "step": 74, "tokens/total": 2385328, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 33792} +{"epoch": 0.29296875, "grad_norm": 0.7465697526931763, "learning_rate": 9.777055165907117e-05, "loss": 0.017319513484835625, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.01747, "step": 75, "tokens/total": 2416416, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 34223} +{"epoch": 0.296875, "grad_norm": 1.1642632484436035, "learning_rate": 9.767942340652993e-05, "loss": 0.02766656130552292, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.02805, "step": 76, "tokens/total": 2446192, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 34649} +{"epoch": 0.30078125, "grad_norm": 0.5575681924819946, "learning_rate": 9.758651908897035e-05, "loss": 0.012136808596551418, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01221, "step": 77, "tokens/total": 2478312, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 35094} +{"epoch": 0.3046875, "grad_norm": 1.2236088514328003, "learning_rate": 9.749184257252033e-05, "loss": 0.05090385302901268, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.05222, "step": 78, "tokens/total": 2509984, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 35534} +{"epoch": 0.30859375, "grad_norm": 1.9397940635681152, "learning_rate": 9.739539779705614e-05, "loss": 0.03581157699227333, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.03646, "step": 79, "tokens/total": 2546536, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 36029} +{"epoch": 0.3125, "grad_norm": 1.5684287548065186, "learning_rate": 9.729718877603861e-05, "loss": 0.014166213572025299, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.01427, "step": 80, "tokens/total": 2578048, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 36469} +{"epoch": 0.31640625, "grad_norm": 1.7710087299346924, "learning_rate": 9.719721959634592e-05, "loss": 0.04042399674654007, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.04125, "step": 81, "tokens/total": 2609640, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 36906} +{"epoch": 0.3203125, "grad_norm": 0.34172675013542175, "learning_rate": 9.709549441810375e-05, "loss": 0.012818311341106892, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.0129, "step": 82, "tokens/total": 2644408, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 37390} +{"epoch": 0.32421875, "grad_norm": 0.5544705390930176, "learning_rate": 9.699201747451195e-05, "loss": 0.0288153737783432, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.02923, "step": 83, "tokens/total": 2675576, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 37838} +{"epoch": 0.328125, "grad_norm": 0.2731646001338959, "learning_rate": 9.688679307166854e-05, "loss": 0.011785678565502167, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.01186, "step": 84, "tokens/total": 2706448, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 38310} +{"epoch": 0.33203125, "grad_norm": 0.6689159274101257, "learning_rate": 9.677982558839042e-05, "loss": 0.032925717532634735, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.03347, "step": 85, "tokens/total": 2738824, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 38789} +{"epoch": 0.3359375, "grad_norm": 1.1869693994522095, "learning_rate": 9.66711194760312e-05, "loss": 0.009798365645110607, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00985, "step": 86, "tokens/total": 2775216, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 39277} +{"epoch": 0.33984375, "grad_norm": 0.6124826669692993, "learning_rate": 9.656067925829593e-05, "loss": 0.01960911974310875, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.0198, "step": 87, "tokens/total": 2804720, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 39737} +{"epoch": 0.34375, "grad_norm": 0.567359209060669, "learning_rate": 9.644850953105288e-05, "loss": 0.014449391514062881, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.01455, "step": 88, "tokens/total": 2833944, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 40179} +{"epoch": 0.34765625, "grad_norm": 0.2417096197605133, "learning_rate": 9.633461496214225e-05, "loss": 0.008306442759931087, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00834, "step": 89, "tokens/total": 2865800, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 40649} +{"epoch": 0.3515625, "grad_norm": 0.5075247883796692, "learning_rate": 9.621900029118195e-05, "loss": 0.016270415857434273, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.0164, "step": 90, "tokens/total": 2894864, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 41080} +{"epoch": 0.35546875, "grad_norm": 0.15182487666606903, "learning_rate": 9.610167032937036e-05, "loss": 0.003865080652758479, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00387, "step": 91, "tokens/total": 2929216, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 41599} +{"epoch": 0.359375, "grad_norm": 0.16264460980892181, "learning_rate": 9.598262995928611e-05, "loss": 0.003939861431717873, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00395, "step": 92, "tokens/total": 2959280, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 42076} +{"epoch": 0.36328125, "grad_norm": 0.2381962388753891, "learning_rate": 9.586188413468492e-05, "loss": 0.009028374217450619, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00907, "step": 93, "tokens/total": 2993456, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 42522} +{"epoch": 0.3671875, "grad_norm": 0.5136604309082031, "learning_rate": 9.57394378802934e-05, "loss": 0.00984522607177496, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00989, "step": 94, "tokens/total": 3025752, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 42974} +{"epoch": 0.37109375, "grad_norm": 0.43333888053894043, "learning_rate": 9.56152962916e-05, "loss": 0.009333346970379353, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.00938, "step": 95, "tokens/total": 3054192, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 43380} +{"epoch": 0.375, "grad_norm": 0.952511191368103, "learning_rate": 9.548946453464296e-05, "loss": 0.0110994353890419, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.01116, "step": 96, "tokens/total": 3087160, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 43865} +{"epoch": 0.37890625, "grad_norm": 0.5407256484031677, "learning_rate": 9.53619478457953e-05, "loss": 0.012274282053112984, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01235, "step": 97, "tokens/total": 3121000, "tokens/train_per_sec_per_gpu": 7.02, "tokens/trainable": 44328} +{"epoch": 0.3828125, "grad_norm": 0.4449124336242676, "learning_rate": 9.523275153154695e-05, "loss": 0.012241464108228683, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.01232, "step": 98, "tokens/total": 3152272, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 44778} +{"epoch": 0.38671875, "grad_norm": 1.4875000715255737, "learning_rate": 9.51018809682839e-05, "loss": 0.028379984200000763, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.02879, "step": 99, "tokens/total": 3185480, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 45225} +{"epoch": 0.390625, "grad_norm": 0.7426877617835999, "learning_rate": 9.49693416020645e-05, "loss": 0.011538694612681866, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.01161, "step": 100, "tokens/total": 3219360, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 45678} +{"epoch": 0.39453125, "grad_norm": 0.2864832580089569, "learning_rate": 9.483513894839276e-05, "loss": 0.0031155580654740334, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00312, "step": 101, "tokens/total": 3251368, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 46125} +{"epoch": 0.3984375, "grad_norm": 0.4527408480644226, "learning_rate": 9.469927859198888e-05, "loss": 0.006753568071871996, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00678, "step": 102, "tokens/total": 3284184, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 46612} +{"epoch": 0.40234375, "grad_norm": 0.29175469279289246, "learning_rate": 9.456176618655689e-05, "loss": 0.011938968673348427, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.01201, "step": 103, "tokens/total": 3314752, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 47059} +{"epoch": 0.40625, "grad_norm": 1.624334454536438, "learning_rate": 9.442260745454927e-05, "loss": 0.0078124129213392735, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00784, "step": 104, "tokens/total": 3349736, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 47536} +{"epoch": 0.41015625, "grad_norm": 0.2851387560367584, "learning_rate": 9.428180818692884e-05, "loss": 0.009097795933485031, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00914, "step": 105, "tokens/total": 3384224, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 48037} +{"epoch": 0.4140625, "grad_norm": 0.393922358751297, "learning_rate": 9.413937424292791e-05, "loss": 0.011979411356151104, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.01205, "step": 106, "tokens/total": 3420808, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 48491} +{"epoch": 0.41796875, "grad_norm": 0.360831618309021, "learning_rate": 9.399531154980424e-05, "loss": 0.01113860122859478, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0112, "step": 107, "tokens/total": 3453512, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 48940} +{"epoch": 0.421875, "grad_norm": 0.40541401505470276, "learning_rate": 9.384962610259455e-05, "loss": 0.008792240172624588, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00883, "step": 108, "tokens/total": 3486192, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 49389} +{"epoch": 0.42578125, "grad_norm": 0.12377746403217316, "learning_rate": 9.370232396386494e-05, "loss": 0.0021212755236774683, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00212, "step": 109, "tokens/total": 3515384, "tokens/train_per_sec_per_gpu": 10.17, "tokens/trainable": 49838} +{"epoch": 0.4296875, "grad_norm": 1.1213675737380981, "learning_rate": 9.355341126345868e-05, "loss": 0.010319423861801624, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01037, "step": 110, "tokens/total": 3548264, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 50310} +{"epoch": 0.43359375, "grad_norm": 0.6072261333465576, "learning_rate": 9.340289419824107e-05, "loss": 0.008354168385267258, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.00839, "step": 111, "tokens/total": 3579288, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 50755} +{"epoch": 0.4375, "grad_norm": 0.5486133098602295, "learning_rate": 9.325077903184159e-05, "loss": 0.01297601219266653, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.01306, "step": 112, "tokens/total": 3611240, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 51213} +{"epoch": 0.44140625, "grad_norm": 0.03824262693524361, "learning_rate": 9.30970720943932e-05, "loss": 0.0002777782501652837, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00028, "step": 113, "tokens/total": 3644136, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 51660} +{"epoch": 0.4453125, "grad_norm": 0.17407506704330444, "learning_rate": 9.2941779782269e-05, "loss": 0.0010768240317702293, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00108, "step": 114, "tokens/total": 3676368, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 52133} +{"epoch": 0.44921875, "grad_norm": 0.287673681974411, "learning_rate": 9.278490855781596e-05, "loss": 0.002094220370054245, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.0021, "step": 115, "tokens/total": 3707896, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 52580} +{"epoch": 0.453125, "grad_norm": 1.2271702289581299, "learning_rate": 9.262646494908604e-05, "loss": 0.016758017241954803, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.0169, "step": 116, "tokens/total": 3742296, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 53033} +{"epoch": 0.45703125, "grad_norm": 0.20044933259487152, "learning_rate": 9.246645554956457e-05, "loss": 0.0012002566363662481, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.0012, "step": 117, "tokens/total": 3776456, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 53517} +{"epoch": 0.4609375, "grad_norm": 3.561727285385132, "learning_rate": 9.230488701789578e-05, "loss": 0.02245260775089264, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.02271, "step": 118, "tokens/total": 3811688, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 53967} +{"epoch": 0.46484375, "grad_norm": 0.036050472408533096, "learning_rate": 9.214176607760577e-05, "loss": 0.0004805333446711302, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00048, "step": 119, "tokens/total": 3847200, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 54430} +{"epoch": 0.46875, "grad_norm": 0.4758164584636688, "learning_rate": 9.197709951682268e-05, "loss": 0.005568540655076504, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.00558, "step": 120, "tokens/total": 3877800, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 54896} +{"epoch": 0.47265625, "grad_norm": 2.2152979373931885, "learning_rate": 9.181089418799428e-05, "loss": 0.02947990410029888, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.02992, "step": 121, "tokens/total": 3910976, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 55379} +{"epoch": 0.4765625, "grad_norm": 0.5870887041091919, "learning_rate": 9.164315700760271e-05, "loss": 0.015310181304812431, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01543, "step": 122, "tokens/total": 3943768, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 55803} +{"epoch": 0.48046875, "grad_norm": 0.25940874218940735, "learning_rate": 9.147389495587671e-05, "loss": 0.00404341658577323, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00405, "step": 123, "tokens/total": 3973816, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 56249} +{"epoch": 0.484375, "grad_norm": 0.6026659607887268, "learning_rate": 9.130311507650116e-05, "loss": 0.00894087366759777, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00898, "step": 124, "tokens/total": 4004704, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 56713} +{"epoch": 0.48828125, "grad_norm": 0.36469766497612, "learning_rate": 9.113082447632394e-05, "loss": 0.006736604496836662, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00676, "step": 125, "tokens/total": 4036040, "tokens/train_per_sec_per_gpu": 10.34, "tokens/trainable": 57196} +{"epoch": 0.4921875, "grad_norm": 0.5277873277664185, "learning_rate": 9.09570303250602e-05, "loss": 0.010322893038392067, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.01038, "step": 126, "tokens/total": 4066792, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 57680} +{"epoch": 0.49609375, "grad_norm": 0.47275227308273315, "learning_rate": 9.078173985499394e-05, "loss": 0.005029073916375637, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00504, "step": 127, "tokens/total": 4096112, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 58110} +{"epoch": 0.5, "grad_norm": 0.5315716862678528, "learning_rate": 9.060496036067713e-05, "loss": 0.01201667357236147, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.01209, "step": 128, "tokens/total": 4131472, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 58534} +{"epoch": 0.50390625, "grad_norm": 0.31068697571754456, "learning_rate": 9.042669919862615e-05, "loss": 0.0035008350387215614, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00351, "step": 129, "tokens/total": 4162912, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 58998} +{"epoch": 0.5078125, "grad_norm": 0.5814250111579895, "learning_rate": 9.024696378701557e-05, "loss": 0.016061628237366676, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.01619, "step": 130, "tokens/total": 4195768, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 59448} +{"epoch": 0.51171875, "grad_norm": 0.27031734585762024, "learning_rate": 9.006576160536948e-05, "loss": 0.005686615128070116, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.0057, "step": 131, "tokens/total": 4227192, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 59906} +{"epoch": 0.515625, "grad_norm": 0.2930522859096527, "learning_rate": 8.988310019425035e-05, "loss": 0.00920164491981268, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00924, "step": 132, "tokens/total": 4263304, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 60350} +{"epoch": 0.51953125, "grad_norm": 0.22769023478031158, "learning_rate": 8.969898715494506e-05, "loss": 0.005831345450133085, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00585, "step": 133, "tokens/total": 4297352, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 60831} +{"epoch": 0.5234375, "grad_norm": 0.44867652654647827, "learning_rate": 8.951343014914869e-05, "loss": 0.010992035269737244, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.01105, "step": 134, "tokens/total": 4332152, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 61305} +{"epoch": 0.52734375, "grad_norm": 0.1645834743976593, "learning_rate": 8.932643689864568e-05, "loss": 0.002451565582305193, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00245, "step": 135, "tokens/total": 4365272, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 61792} +{"epoch": 0.53125, "grad_norm": 0.09671884030103683, "learning_rate": 8.913801518498845e-05, "loss": 0.0014303313801065087, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00143, "step": 136, "tokens/total": 4399928, "tokens/train_per_sec_per_gpu": 6.81, "tokens/trainable": 62253} +{"epoch": 0.53515625, "grad_norm": 0.0499332956969738, "learning_rate": 8.894817284917364e-05, "loss": 0.0009391457424499094, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00094, "step": 137, "tokens/total": 4434336, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 62707} +{"epoch": 0.5390625, "grad_norm": 0.03112562745809555, "learning_rate": 8.875691779131569e-05, "loss": 0.00042099624988622963, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00042, "step": 138, "tokens/total": 4468872, "tokens/train_per_sec_per_gpu": 6.31, "tokens/trainable": 63145} +{"epoch": 0.54296875, "grad_norm": 0.33777058124542236, "learning_rate": 8.856425797031829e-05, "loss": 0.003038874827325344, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00304, "step": 139, "tokens/total": 4502416, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 63587} +{"epoch": 0.546875, "grad_norm": 0.6313179135322571, "learning_rate": 8.837020140354295e-05, "loss": 0.012149406597018242, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.01222, "step": 140, "tokens/total": 4536024, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 64052} +{"epoch": 0.55078125, "grad_norm": 1.021543264389038, "learning_rate": 8.817475616647554e-05, "loss": 0.009159469977021217, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.0092, "step": 141, "tokens/total": 4568960, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 64526} +{"epoch": 0.5546875, "grad_norm": 0.6649272441864014, "learning_rate": 8.797793039239017e-05, "loss": 0.016962455585598946, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.01711, "step": 142, "tokens/total": 4601656, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 64983} +{"epoch": 0.55859375, "grad_norm": 0.32206371426582336, "learning_rate": 8.777973227201069e-05, "loss": 0.0013662994606420398, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00137, "step": 143, "tokens/total": 4636136, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 65492} +{"epoch": 0.5625, "grad_norm": 0.2489043027162552, "learning_rate": 8.758017005316988e-05, "loss": 0.008389117196202278, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00842, "step": 144, "tokens/total": 4670160, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 65925} +{"epoch": 0.56640625, "grad_norm": 0.5617746710777283, "learning_rate": 8.737925204046629e-05, "loss": 0.00411451980471611, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.00412, "step": 145, "tokens/total": 4703792, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 66383} +{"epoch": 0.5703125, "grad_norm": 0.6041951179504395, "learning_rate": 8.717698659491851e-05, "loss": 0.015162218362092972, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.01528, "step": 146, "tokens/total": 4737640, "tokens/train_per_sec_per_gpu": 6.95, "tokens/trainable": 66840} +{"epoch": 0.57421875, "grad_norm": 0.9016480445861816, "learning_rate": 8.697338213361735e-05, "loss": 0.014017830602824688, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01412, "step": 147, "tokens/total": 4770704, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 67297} +{"epoch": 0.578125, "grad_norm": 0.7610868811607361, "learning_rate": 8.676844712937552e-05, "loss": 0.008894921280443668, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00893, "step": 148, "tokens/total": 4807376, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 67769} +{"epoch": 0.58203125, "grad_norm": 1.0707520246505737, "learning_rate": 8.656219011037509e-05, "loss": 0.024141807109117508, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.02444, "step": 149, "tokens/total": 4840728, "tokens/train_per_sec_per_gpu": 6.54, "tokens/trainable": 68257} +{"epoch": 0.5859375, "grad_norm": 0.4729861319065094, "learning_rate": 8.63546196598125e-05, "loss": 0.004918929189443588, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00493, "step": 150, "tokens/total": 4873880, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 68706} +{"epoch": 0.58984375, "grad_norm": 0.41318291425704956, "learning_rate": 8.614574441554145e-05, "loss": 0.015610345639288425, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.01573, "step": 151, "tokens/total": 4904512, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 69131} +{"epoch": 0.59375, "grad_norm": 0.3427720367908478, "learning_rate": 8.593557306971349e-05, "loss": 0.0072993808425962925, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00733, "step": 152, "tokens/total": 4937048, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 69586} +{"epoch": 0.59765625, "grad_norm": 0.7849957346916199, "learning_rate": 8.572411436841618e-05, "loss": 0.021099617704749107, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.02132, "step": 153, "tokens/total": 4968224, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 70061} +{"epoch": 0.6015625, "grad_norm": 0.4592441916465759, "learning_rate": 8.551137711130922e-05, "loss": 0.021112680435180664, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.02134, "step": 154, "tokens/total": 4997272, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 70467} +{"epoch": 0.60546875, "grad_norm": 0.17997413873672485, "learning_rate": 8.529737015125824e-05, "loss": 0.005319032818078995, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00533, "step": 155, "tokens/total": 5031072, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 70933} +{"epoch": 0.609375, "grad_norm": 0.3908715546131134, "learning_rate": 8.508210239396639e-05, "loss": 0.015039256773889065, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.01515, "step": 156, "tokens/total": 5063808, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 71382} +{"epoch": 0.61328125, "grad_norm": 0.07480531930923462, "learning_rate": 8.486558279760375e-05, "loss": 0.0025357212871313095, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00254, "step": 157, "tokens/total": 5094032, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 71808} +{"epoch": 0.6171875, "grad_norm": 0.2094024419784546, "learning_rate": 8.464782037243449e-05, "loss": 0.0051111155189573765, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00512, "step": 158, "tokens/total": 5125664, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 72249} +{"epoch": 0.62109375, "grad_norm": 0.28858035802841187, "learning_rate": 8.442882418044202e-05, "loss": 0.013800648972392082, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.0139, "step": 159, "tokens/total": 5156952, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 72726} +{"epoch": 0.625, "grad_norm": 0.11017389595508575, "learning_rate": 8.420860333495179e-05, "loss": 0.0042923265136778355, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.0043, "step": 160, "tokens/total": 5187120, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 73148} +{"epoch": 0.62890625, "grad_norm": 0.18401522934436798, "learning_rate": 8.398716700025208e-05, "loss": 0.004814868327230215, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00483, "step": 161, "tokens/total": 5219080, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 73600} +{"epoch": 0.6328125, "grad_norm": 0.3105522096157074, "learning_rate": 8.376452439121266e-05, "loss": 0.013436454348266125, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.01353, "step": 162, "tokens/total": 5248680, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 74068} +{"epoch": 0.63671875, "grad_norm": 0.24305610358715057, "learning_rate": 8.354068477290124e-05, "loss": 0.004349930211901665, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00436, "step": 163, "tokens/total": 5280104, "tokens/train_per_sec_per_gpu": 10.01, "tokens/trainable": 74527} +{"epoch": 0.640625, "grad_norm": 0.06493128091096878, "learning_rate": 8.331565746019807e-05, "loss": 0.001294412650167942, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.0013, "step": 164, "tokens/total": 5311328, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 74992} +{"epoch": 0.64453125, "grad_norm": 1.257920265197754, "learning_rate": 8.308945181740812e-05, "loss": 0.027397722005844116, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.02778, "step": 165, "tokens/total": 5343960, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 75442} +{"epoch": 0.6484375, "grad_norm": 0.1984734684228897, "learning_rate": 8.286207725787153e-05, "loss": 0.006222587078809738, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00624, "step": 166, "tokens/total": 5377312, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 75887} +{"epoch": 0.65234375, "grad_norm": 0.41656503081321716, "learning_rate": 8.263354324357182e-05, "loss": 0.009777607396245003, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00983, "step": 167, "tokens/total": 5409824, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 76331} +{"epoch": 0.65625, "grad_norm": 1.0027254819869995, "learning_rate": 8.240385928474219e-05, "loss": 0.020352747291326523, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.02056, "step": 168, "tokens/total": 5440904, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 76745} +{"epoch": 0.66015625, "grad_norm": 0.10981178283691406, "learning_rate": 8.217303493946967e-05, "loss": 0.0009488011128269136, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00095, "step": 169, "tokens/total": 5474920, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 77201} +{"epoch": 0.6640625, "grad_norm": 0.2863771319389343, "learning_rate": 8.194107981329746e-05, "loss": 0.004469533916562796, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00448, "step": 170, "tokens/total": 5511856, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 77655} +{"epoch": 0.66796875, "grad_norm": 0.008555101230740547, "learning_rate": 8.170800355882518e-05, "loss": 0.00016906496603041887, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00017, "step": 171, "tokens/total": 5544104, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 78122} +{"epoch": 0.671875, "grad_norm": 0.23804971575737, "learning_rate": 8.147381587530713e-05, "loss": 0.002435237867757678, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00244, "step": 172, "tokens/total": 5577344, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 78576} +{"epoch": 0.67578125, "grad_norm": 0.8725315928459167, "learning_rate": 8.123852650824877e-05, "loss": 0.007513358257710934, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00754, "step": 173, "tokens/total": 5610024, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 79059} +{"epoch": 0.6796875, "grad_norm": 0.09769926965236664, "learning_rate": 8.100214524900103e-05, "loss": 0.001141593442298472, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00114, "step": 174, "tokens/total": 5643184, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 79498} +{"epoch": 0.68359375, "grad_norm": 0.08715018630027771, "learning_rate": 8.076468193435301e-05, "loss": 0.001286888960748911, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00129, "step": 175, "tokens/total": 5672824, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 79922} +{"epoch": 0.6875, "grad_norm": 0.11032000184059143, "learning_rate": 8.052614644612253e-05, "loss": 0.0017335417214781046, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00174, "step": 176, "tokens/total": 5705576, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 80383} +{"epoch": 0.69140625, "grad_norm": 0.01673627272248268, "learning_rate": 8.028654871074489e-05, "loss": 0.00023250632511917502, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00023, "step": 177, "tokens/total": 5737840, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 80824} +{"epoch": 0.6953125, "grad_norm": 0.061986472457647324, "learning_rate": 8.004589869885986e-05, "loss": 0.0006494428962469101, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00065, "step": 178, "tokens/total": 5767552, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 81243} +{"epoch": 0.69921875, "grad_norm": 0.4850742518901825, "learning_rate": 7.980420642489674e-05, "loss": 0.011705511249601841, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01177, "step": 179, "tokens/total": 5802512, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 81716} +{"epoch": 0.703125, "grad_norm": 0.02727707475423813, "learning_rate": 7.95614819466576e-05, "loss": 0.00025580779765732586, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00026, "step": 180, "tokens/total": 5834984, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 82181} +{"epoch": 0.70703125, "grad_norm": 0.14892250299453735, "learning_rate": 7.931773536489872e-05, "loss": 0.0023895937483757734, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00239, "step": 181, "tokens/total": 5867336, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 82626} +{"epoch": 0.7109375, "grad_norm": 1.0406994819641113, "learning_rate": 7.907297682291035e-05, "loss": 0.008219324983656406, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00825, "step": 182, "tokens/total": 5899232, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 83089} +{"epoch": 0.71484375, "grad_norm": 0.11097357422113419, "learning_rate": 7.882721650609442e-05, "loss": 0.0016687392489984632, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00167, "step": 183, "tokens/total": 5934816, "tokens/train_per_sec_per_gpu": 6.65, "tokens/trainable": 83590} +{"epoch": 0.71875, "grad_norm": 0.08020482212305069, "learning_rate": 7.85804646415409e-05, "loss": 0.0015673314919695258, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.00157, "step": 184, "tokens/total": 5968120, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 84046} +{"epoch": 0.72265625, "grad_norm": 0.5228198766708374, "learning_rate": 7.833273149760207e-05, "loss": 0.0014708650996908545, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00147, "step": 185, "tokens/total": 5998600, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 84517} +{"epoch": 0.7265625, "grad_norm": 0.7486904263496399, "learning_rate": 7.808402738346527e-05, "loss": 0.011666962876915932, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01174, "step": 186, "tokens/total": 6033832, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 84974} +{"epoch": 0.73046875, "grad_norm": 0.2625448405742645, "learning_rate": 7.783436264872382e-05, "loss": 0.0025151504669338465, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00252, "step": 187, "tokens/total": 6068192, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 85460} +{"epoch": 0.734375, "grad_norm": 0.7006391882896423, "learning_rate": 7.758374768294647e-05, "loss": 0.0050743441097438335, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00509, "step": 188, "tokens/total": 6101464, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 85939} +{"epoch": 0.73828125, "grad_norm": 0.3009427487850189, "learning_rate": 7.733219291524489e-05, "loss": 0.005974139552563429, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00599, "step": 189, "tokens/total": 6133008, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 86377} +{"epoch": 0.7421875, "grad_norm": 0.32584574818611145, "learning_rate": 7.707970881383977e-05, "loss": 0.0020120195113122463, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.00201, "step": 190, "tokens/total": 6163048, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 86834} +{"epoch": 0.74609375, "grad_norm": 1.3102848529815674, "learning_rate": 7.682630588562518e-05, "loss": 0.014433969743549824, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01454, "step": 191, "tokens/total": 6196072, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 87265} +{"epoch": 0.75, "grad_norm": 0.5453954935073853, "learning_rate": 7.657199467573129e-05, "loss": 0.010124867781996727, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01018, "step": 192, "tokens/total": 6227312, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 87747} +{"epoch": 0.75390625, "grad_norm": 0.022446345537900925, "learning_rate": 7.631678576708561e-05, "loss": 0.00028951468993909657, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00029, "step": 193, "tokens/total": 6260240, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 88239} +{"epoch": 0.7578125, "grad_norm": 0.13936300575733185, "learning_rate": 7.606068977997255e-05, "loss": 0.002437296323478222, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00244, "step": 194, "tokens/total": 6294280, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 88718} +{"epoch": 0.76171875, "grad_norm": 0.819054126739502, "learning_rate": 7.580371737159148e-05, "loss": 0.028680739924311638, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.0291, "step": 195, "tokens/total": 6323368, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 89129} +{"epoch": 0.765625, "grad_norm": 0.13420511782169342, "learning_rate": 7.554587923561324e-05, "loss": 0.0017238017171621323, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00173, "step": 196, "tokens/total": 6354960, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 89567} +{"epoch": 0.76953125, "grad_norm": 0.45732519030570984, "learning_rate": 7.528718610173511e-05, "loss": 0.009214911609888077, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.00926, "step": 197, "tokens/total": 6387208, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 89988} +{"epoch": 0.7734375, "grad_norm": 0.28948864340782166, "learning_rate": 7.502764873523431e-05, "loss": 0.0052129002287983894, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00523, "step": 198, "tokens/total": 6418720, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 90434} +{"epoch": 0.77734375, "grad_norm": 1.0151264667510986, "learning_rate": 7.476727793652011e-05, "loss": 0.014733325690031052, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.01484, "step": 199, "tokens/total": 6449184, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 90913} +{"epoch": 0.78125, "grad_norm": 0.31125912070274353, "learning_rate": 7.450608454068415e-05, "loss": 0.009561792016029358, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00961, "step": 200, "tokens/total": 6481824, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 91355} +{"epoch": 0.78515625, "grad_norm": 0.3962952196598053, "learning_rate": 7.424407941704987e-05, "loss": 0.010906578041613102, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.01097, "step": 201, "tokens/total": 6515896, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 91856} +{"epoch": 0.7890625, "grad_norm": 0.7069784998893738, "learning_rate": 7.398127346871986e-05, "loss": 0.032930999994277954, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.03348, "step": 202, "tokens/total": 6548784, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 92311} +{"epoch": 0.79296875, "grad_norm": 0.05219624191522598, "learning_rate": 7.371767763212238e-05, "loss": 0.0009129209793172777, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00091, "step": 203, "tokens/total": 6580256, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 92764} +{"epoch": 0.796875, "grad_norm": 0.42366325855255127, "learning_rate": 7.345330287655617e-05, "loss": 0.002562811831012368, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00257, "step": 204, "tokens/total": 6613976, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 93227} +{"epoch": 0.80078125, "grad_norm": 0.21955475211143494, "learning_rate": 7.31881602037339e-05, "loss": 0.008023942820727825, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.00806, "step": 205, "tokens/total": 6644552, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 93675} +{"epoch": 0.8046875, "grad_norm": 0.18456289172172546, "learning_rate": 7.29222606473245e-05, "loss": 0.002876901999115944, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00288, "step": 206, "tokens/total": 6675056, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 94120} +{"epoch": 0.80859375, "grad_norm": 0.40815746784210205, "learning_rate": 7.265561527249383e-05, "loss": 0.013485540635883808, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.01358, "step": 207, "tokens/total": 6706624, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 94557} +{"epoch": 0.8125, "grad_norm": 0.2730261981487274, "learning_rate": 7.238823517544436e-05, "loss": 0.01279991865158081, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01288, "step": 208, "tokens/total": 6741008, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 95035} +{"epoch": 0.81640625, "grad_norm": 0.10115110129117966, "learning_rate": 7.212013148295333e-05, "loss": 0.003786234650760889, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00379, "step": 209, "tokens/total": 6774872, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 95517} +{"epoch": 0.8203125, "grad_norm": 0.1987261325120926, "learning_rate": 7.185131535190975e-05, "loss": 0.0040604169480502605, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00407, "step": 210, "tokens/total": 6805816, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 96026} +{"epoch": 0.82421875, "grad_norm": 0.2383221536874771, "learning_rate": 7.158179796885005e-05, "loss": 0.007446107920259237, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00747, "step": 211, "tokens/total": 6837592, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 96477} +{"epoch": 0.828125, "grad_norm": 0.4146282374858856, "learning_rate": 7.131159054949273e-05, "loss": 0.005652093328535557, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00567, "step": 212, "tokens/total": 6874384, "tokens/train_per_sec_per_gpu": 6.98, "tokens/trainable": 96951} +{"epoch": 0.83203125, "grad_norm": 0.3096284568309784, "learning_rate": 7.104070433827139e-05, "loss": 0.003771107643842697, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00378, "step": 213, "tokens/total": 6905872, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 97350} +{"epoch": 0.8359375, "grad_norm": 0.26115795969963074, "learning_rate": 7.076915060786705e-05, "loss": 0.004884599708020687, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.0049, "step": 214, "tokens/total": 6937984, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 97792} +{"epoch": 0.83984375, "grad_norm": 0.15413039922714233, "learning_rate": 7.049694065873882e-05, "loss": 0.002133278176188469, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00214, "step": 215, "tokens/total": 6970704, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 98257} +{"epoch": 0.84375, "grad_norm": 0.8247398138046265, "learning_rate": 7.022408581865382e-05, "loss": 0.005758584942668676, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00578, "step": 216, "tokens/total": 7004616, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 98731} +{"epoch": 0.84765625, "grad_norm": 0.04199913144111633, "learning_rate": 6.99505974422157e-05, "loss": 0.0005579755525104702, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00056, "step": 217, "tokens/total": 7036880, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 99212} +{"epoch": 0.8515625, "grad_norm": 0.6541374921798706, "learning_rate": 6.967648691039213e-05, "loss": 0.0004925322718918324, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00049, "step": 218, "tokens/total": 7068208, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 99654} +{"epoch": 0.85546875, "grad_norm": 0.03978859260678291, "learning_rate": 6.940176563004123e-05, "loss": 0.00047738294233568013, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00048, "step": 219, "tokens/total": 7098112, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 100086} +{"epoch": 0.859375, "grad_norm": 0.44381341338157654, "learning_rate": 6.912644503343682e-05, "loss": 0.005951987113803625, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00597, "step": 220, "tokens/total": 7132432, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 100524} +{"epoch": 0.86328125, "grad_norm": 0.06032999977469444, "learning_rate": 6.885053657779273e-05, "loss": 0.00045497194514609873, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.00046, "step": 221, "tokens/total": 7164016, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 100996} +{"epoch": 0.8671875, "grad_norm": 0.010105278342962265, "learning_rate": 6.857405174478604e-05, "loss": 0.00012325441639404744, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00012, "step": 222, "tokens/total": 7198344, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 101449} +{"epoch": 0.87109375, "grad_norm": 0.028267106041312218, "learning_rate": 6.82970020400792e-05, "loss": 0.00020678855071309954, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00021, "step": 223, "tokens/total": 7232192, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 101905} +{"epoch": 0.875, "grad_norm": 0.016877712681889534, "learning_rate": 6.801939899284132e-05, "loss": 0.00013587932335212827, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00014, "step": 224, "tokens/total": 7266984, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 102411} +{"epoch": 0.87890625, "grad_norm": 0.06818168610334396, "learning_rate": 6.774125415526827e-05, "loss": 8.457632065983489e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00008, "step": 225, "tokens/total": 7299512, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 102882} +{"epoch": 0.8828125, "grad_norm": 0.17277655005455017, "learning_rate": 6.746257910210214e-05, "loss": 0.0005133182276040316, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00051, "step": 226, "tokens/total": 7331120, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 103332} +{"epoch": 0.88671875, "grad_norm": 0.38574397563934326, "learning_rate": 6.718338543014937e-05, "loss": 0.00939441192895174, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00944, "step": 227, "tokens/total": 7360504, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 103790} +{"epoch": 0.890625, "grad_norm": 0.44912058115005493, "learning_rate": 6.69036847577983e-05, "loss": 0.0010282598668709397, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00103, "step": 228, "tokens/total": 7390760, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 104246} +{"epoch": 0.89453125, "grad_norm": 0.07154402136802673, "learning_rate": 6.662348872453553e-05, "loss": 0.0002389491128269583, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00024, "step": 229, "tokens/total": 7423352, "tokens/train_per_sec_per_gpu": 10.32, "tokens/trainable": 104707} +{"epoch": 0.8984375, "grad_norm": 0.7018928527832031, "learning_rate": 6.63428089904618e-05, "loss": 0.009487633593380451, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00953, "step": 230, "tokens/total": 7457152, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 105167} +{"epoch": 0.90234375, "grad_norm": 0.1413300782442093, "learning_rate": 6.60616572358065e-05, "loss": 9.509777737548575e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.0001, "step": 231, "tokens/total": 7488648, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 105576} +{"epoch": 0.90625, "grad_norm": 0.9472690224647522, "learning_rate": 6.578004516044172e-05, "loss": 0.0005619538715109229, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00056, "step": 232, "tokens/total": 7522160, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 106043} +{"epoch": 0.91015625, "grad_norm": 0.8581891059875488, "learning_rate": 6.549798448339548e-05, "loss": 0.006628398783504963, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00665, "step": 233, "tokens/total": 7557336, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 106506} +{"epoch": 0.9140625, "grad_norm": 1.1150047779083252, "learning_rate": 6.521548694236384e-05, "loss": 0.006766194943338633, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00679, "step": 234, "tokens/total": 7590432, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 106951} +{"epoch": 0.91796875, "grad_norm": 1.0135712623596191, "learning_rate": 6.493256429322259e-05, "loss": 0.012543803080916405, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.01262, "step": 235, "tokens/total": 7623240, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 107382} +{"epoch": 0.921875, "grad_norm": 0.43076780438423157, "learning_rate": 6.464922830953799e-05, "loss": 0.002130216918885708, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00213, "step": 236, "tokens/total": 7657968, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 107814} +{"epoch": 0.92578125, "grad_norm": 0.5501914024353027, "learning_rate": 6.436549078207688e-05, "loss": 0.00888830702751875, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00893, "step": 237, "tokens/total": 7689320, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 108274} +{"epoch": 0.9296875, "grad_norm": 0.4833866059780121, "learning_rate": 6.408136351831592e-05, "loss": 0.006164690479636192, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00618, "step": 238, "tokens/total": 7721568, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 108714} +{"epoch": 0.93359375, "grad_norm": 0.4253029227256775, "learning_rate": 6.379685834195036e-05, "loss": 0.004488594364374876, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.0045, "step": 239, "tokens/total": 7756464, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 109220} +{"epoch": 0.9375, "grad_norm": 0.6595520973205566, "learning_rate": 6.351198709240186e-05, "loss": 0.01107285637408495, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.01113, "step": 240, "tokens/total": 7791616, "tokens/train_per_sec_per_gpu": 6.91, "tokens/trainable": 109672} +{"epoch": 0.94140625, "grad_norm": 0.5375209450721741, "learning_rate": 6.32267616243259e-05, "loss": 0.017229566350579262, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.01738, "step": 241, "tokens/total": 7823520, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 110135} +{"epoch": 0.9453125, "grad_norm": 0.05699203535914421, "learning_rate": 6.294119380711849e-05, "loss": 0.0011251458199694753, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00113, "step": 242, "tokens/total": 7854424, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 110563} +{"epoch": 0.94921875, "grad_norm": 1.3414511680603027, "learning_rate": 6.265529552442209e-05, "loss": 0.014298874884843826, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.0144, "step": 243, "tokens/total": 7887160, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 111049} +{"epoch": 0.953125, "grad_norm": 0.2718977630138397, "learning_rate": 6.236907867363127e-05, "loss": 0.007284311577677727, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00731, "step": 244, "tokens/total": 7918736, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 111495} +{"epoch": 0.95703125, "grad_norm": 0.44348809123039246, "learning_rate": 6.208255516539749e-05, "loss": 0.011748313903808594, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.01182, "step": 245, "tokens/total": 7949448, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 111968} +{"epoch": 0.9609375, "grad_norm": 0.16032710671424866, "learning_rate": 6.179573692313344e-05, "loss": 0.002791972830891609, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.0028, "step": 246, "tokens/total": 7984680, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 112416} +{"epoch": 0.96484375, "grad_norm": 0.20096628367900848, "learning_rate": 6.150863588251694e-05, "loss": 0.003929967060685158, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00394, "step": 247, "tokens/total": 8019816, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 112882} +{"epoch": 0.96875, "grad_norm": 0.24361419677734375, "learning_rate": 6.122126399099419e-05, "loss": 0.0027426735032349825, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00275, "step": 248, "tokens/total": 8056272, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 113390} +{"epoch": 0.97265625, "grad_norm": 0.7109816074371338, "learning_rate": 6.0933633207282615e-05, "loss": 0.016688985750079155, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.01683, "step": 249, "tokens/total": 8090224, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 113904} +{"epoch": 0.9765625, "grad_norm": 0.2656702399253845, "learning_rate": 6.064575550087316e-05, "loss": 0.007506938185542822, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00754, "step": 250, "tokens/total": 8124528, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 114351} +{"epoch": 0.98046875, "grad_norm": 0.1921294778585434, "learning_rate": 6.0357642851532245e-05, "loss": 0.003312553046271205, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00332, "step": 251, "tokens/total": 8159688, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 114842} +{"epoch": 0.984375, "grad_norm": 0.08827173709869385, "learning_rate": 6.0069307248803294e-05, "loss": 0.0024009174667298794, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.0024, "step": 252, "tokens/total": 8191800, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 115263} +{"epoch": 0.98828125, "grad_norm": 0.10989273339509964, "learning_rate": 5.9780760691507635e-05, "loss": 0.003152113873511553, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00316, "step": 253, "tokens/total": 8221752, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 115703} +{"epoch": 0.9921875, "grad_norm": 0.36964938044548035, "learning_rate": 5.9492015187245334e-05, "loss": 0.015153195708990097, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.01527, "step": 254, "tokens/total": 8255648, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 116157} +{"epoch": 0.99609375, "grad_norm": 0.3890700042247772, "learning_rate": 5.920308275189541e-05, "loss": 0.010535143315792084, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.01059, "step": 255, "tokens/total": 8286664, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 116567} +{"epoch": 1.0, "grad_norm": 0.09425398707389832, "learning_rate": 5.8913975409115874e-05, "loss": 0.0019343059975653887, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00194, "step": 256, "tokens/total": 8318368, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 117030} +{"epoch": 1.00390625, "grad_norm": 0.08699461817741394, "learning_rate": 5.8624705189843395e-05, "loss": 0.0012551277177408338, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00126, "step": 257, "tokens/total": 8353336, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 117517} +{"epoch": 1.0078125, "grad_norm": 0.046132929623126984, "learning_rate": 5.833528413179249e-05, "loss": 0.0010391753166913986, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00104, "step": 258, "tokens/total": 8389112, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 118015} +{"epoch": 1.01171875, "grad_norm": 0.20356953144073486, "learning_rate": 5.80457242789548e-05, "loss": 0.0015223361551761627, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00152, "step": 259, "tokens/total": 8426000, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 118520} +{"epoch": 1.015625, "grad_norm": 0.12513507902622223, "learning_rate": 5.77560376810977e-05, "loss": 0.0019832863472402096, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.00199, "step": 260, "tokens/total": 8458568, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 118992} +{"epoch": 1.01953125, "grad_norm": 0.1621248871088028, "learning_rate": 5.7466236393263005e-05, "loss": 0.003168326336890459, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00317, "step": 261, "tokens/total": 8490840, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 119438} +{"epoch": 1.0234375, "grad_norm": 0.06423820555210114, "learning_rate": 5.717633247526522e-05, "loss": 0.0007080489885993302, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00071, "step": 262, "tokens/total": 8521792, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 119881} +{"epoch": 1.02734375, "grad_norm": 0.4390539824962616, "learning_rate": 5.688633799118971e-05, "loss": 0.0036153914406895638, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00362, "step": 263, "tokens/total": 8551336, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 120285} +{"epoch": 1.03125, "grad_norm": 0.020125409588217735, "learning_rate": 5.659626500889066e-05, "loss": 0.0003844808379653841, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00038, "step": 264, "tokens/total": 8579872, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 120755} +{"epoch": 1.03515625, "grad_norm": 0.2015024721622467, "learning_rate": 5.6306125599488905e-05, "loss": 0.002311549847945571, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00231, "step": 265, "tokens/total": 8608920, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 121194} +{"epoch": 1.0390625, "grad_norm": 0.18365740776062012, "learning_rate": 5.601593183686955e-05, "loss": 0.0019365003099665046, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00194, "step": 266, "tokens/total": 8643600, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 121670} +{"epoch": 1.04296875, "grad_norm": 0.1425224244594574, "learning_rate": 5.572569579717961e-05, "loss": 0.0014775969320908189, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00148, "step": 267, "tokens/total": 8676032, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 122142} +{"epoch": 1.046875, "grad_norm": 0.003777294885367155, "learning_rate": 5.543542955832538e-05, "loss": 7.580849342048168e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00008, "step": 268, "tokens/total": 8713240, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 122585} +{"epoch": 1.05078125, "grad_norm": 0.25997865200042725, "learning_rate": 5.514514519946986e-05, "loss": 0.0034666128922253847, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00347, "step": 269, "tokens/total": 8751664, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 123091} +{"epoch": 1.0546875, "grad_norm": 0.022657284513115883, "learning_rate": 5.485485480053015e-05, "loss": 0.0003057275025639683, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00031, "step": 270, "tokens/total": 8786112, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 123505} +{"epoch": 1.05859375, "grad_norm": 0.08286740630865097, "learning_rate": 5.4564570441674645e-05, "loss": 0.0004591561446432024, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00046, "step": 271, "tokens/total": 8816920, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 123953} +{"epoch": 1.0625, "grad_norm": 0.00530537823215127, "learning_rate": 5.42743042028204e-05, "loss": 7.280247518792748e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00007, "step": 272, "tokens/total": 8851408, "tokens/train_per_sec_per_gpu": 6.7, "tokens/trainable": 124400} +{"epoch": 1.06640625, "grad_norm": 0.007615187205374241, "learning_rate": 5.3984068163130464e-05, "loss": 0.00012365823204163462, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00012, "step": 273, "tokens/total": 8883768, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 124870} +{"epoch": 1.0703125, "grad_norm": 0.5953271985054016, "learning_rate": 5.369387440051111e-05, "loss": 0.014265553094446659, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.01437, "step": 274, "tokens/total": 8916200, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 125333} +{"epoch": 1.07421875, "grad_norm": 0.0028386034537106752, "learning_rate": 5.340373499110935e-05, "loss": 6.472120003309101e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00006, "step": 275, "tokens/total": 8949920, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 125834} +{"epoch": 1.078125, "grad_norm": 0.005639367271214724, "learning_rate": 5.3113662008810304e-05, "loss": 5.876973591512069e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00006, "step": 276, "tokens/total": 8986448, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 126316} +{"epoch": 1.08203125, "grad_norm": 0.031811293214559555, "learning_rate": 5.282366752473479e-05, "loss": 0.00021430753986351192, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00021, "step": 277, "tokens/total": 9020600, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 126798} +{"epoch": 1.0859375, "grad_norm": 0.002791851991787553, "learning_rate": 5.2533763606737005e-05, "loss": 6.345887231873348e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00006, "step": 278, "tokens/total": 9056040, "tokens/train_per_sec_per_gpu": 6.77, "tokens/trainable": 127223} +{"epoch": 1.08984375, "grad_norm": 0.08905360102653503, "learning_rate": 5.224396231890232e-05, "loss": 0.0002363049570703879, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00024, "step": 279, "tokens/total": 9087072, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 127672} +{"epoch": 1.09375, "grad_norm": 0.04024166241288185, "learning_rate": 5.195427572104522e-05, "loss": 0.0004648217000067234, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00046, "step": 280, "tokens/total": 9118376, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 128116} +{"epoch": 1.09765625, "grad_norm": 0.25256723165512085, "learning_rate": 5.166471586820751e-05, "loss": 0.005202850792557001, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00522, "step": 281, "tokens/total": 9152704, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 128589} +{"epoch": 1.1015625, "grad_norm": 0.44299331307411194, "learning_rate": 5.1375294810156615e-05, "loss": 0.008466776460409164, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.0085, "step": 282, "tokens/total": 9184448, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 129036} +{"epoch": 1.10546875, "grad_norm": 0.01707316003739834, "learning_rate": 5.1086024590884144e-05, "loss": 0.0002318450715392828, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00023, "step": 283, "tokens/total": 9217392, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 129485} +{"epoch": 1.109375, "grad_norm": 0.026893965899944305, "learning_rate": 5.079691724810461e-05, "loss": 0.00024931441294029355, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00025, "step": 284, "tokens/total": 9252016, "tokens/train_per_sec_per_gpu": 6.97, "tokens/trainable": 129920} +{"epoch": 1.11328125, "grad_norm": 0.006749930325895548, "learning_rate": 5.0507984812754684e-05, "loss": 7.787322101648897e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00008, "step": 285, "tokens/total": 9284008, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 130417} +{"epoch": 1.1171875, "grad_norm": 0.0170150026679039, "learning_rate": 5.021923930849237e-05, "loss": 0.0001550116139696911, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00016, "step": 286, "tokens/total": 9315984, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 130903} +{"epoch": 1.12109375, "grad_norm": 0.13661602139472961, "learning_rate": 4.99306927511967e-05, "loss": 0.00043790339259430766, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00044, "step": 287, "tokens/total": 9347376, "tokens/train_per_sec_per_gpu": 10.23, "tokens/trainable": 131343} +{"epoch": 1.125, "grad_norm": 0.03339556232094765, "learning_rate": 4.964235714846775e-05, "loss": 7.564769475720823e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00008, "step": 288, "tokens/total": 9377424, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 131763} +{"epoch": 1.12890625, "grad_norm": 0.0016048344550654292, "learning_rate": 4.9354244499126866e-05, "loss": 2.7605472496361472e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00003, "step": 289, "tokens/total": 9408104, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 132229} +{"epoch": 1.1328125, "grad_norm": 0.012041190639138222, "learning_rate": 4.90663667927174e-05, "loss": 0.0001955874904524535, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.0002, "step": 290, "tokens/total": 9439944, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 132678} +{"epoch": 1.13671875, "grad_norm": 0.2239098995923996, "learning_rate": 4.877873600900581e-05, "loss": 0.0012927300995215774, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00129, "step": 291, "tokens/total": 9472104, "tokens/train_per_sec_per_gpu": 6.11, "tokens/trainable": 133136} +{"epoch": 1.140625, "grad_norm": 0.0019722268916666508, "learning_rate": 4.849136411748306e-05, "loss": 4.993486072635278e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00005, "step": 292, "tokens/total": 9508336, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 133608} +{"epoch": 1.14453125, "grad_norm": 0.35881558060646057, "learning_rate": 4.8204263076866574e-05, "loss": 0.0031016217544674873, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00311, "step": 293, "tokens/total": 9541552, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 134108} +{"epoch": 1.1484375, "grad_norm": 0.001312487991526723, "learning_rate": 4.791744483460251e-05, "loss": 2.849481643352192e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00003, "step": 294, "tokens/total": 9575072, "tokens/train_per_sec_per_gpu": 6.47, "tokens/trainable": 134541} +{"epoch": 1.15234375, "grad_norm": 0.029786111786961555, "learning_rate": 4.7630921326368736e-05, "loss": 0.0002470850886311382, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00025, "step": 295, "tokens/total": 9610344, "tokens/train_per_sec_per_gpu": 6.31, "tokens/trainable": 134966} +{"epoch": 1.15625, "grad_norm": 0.06275834888219833, "learning_rate": 4.7344704475577916e-05, "loss": 0.0006941219326108694, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00069, "step": 296, "tokens/total": 9639728, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 135402} +{"epoch": 1.16015625, "grad_norm": 0.0479314960539341, "learning_rate": 4.705880619288153e-05, "loss": 0.00036879454273730516, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00037, "step": 297, "tokens/total": 9671424, "tokens/train_per_sec_per_gpu": 6.96, "tokens/trainable": 135804} +{"epoch": 1.1640625, "grad_norm": 0.017595937475562096, "learning_rate": 4.677323837567412e-05, "loss": 0.00024294947797898203, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00024, "step": 298, "tokens/total": 9700808, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 136231} +{"epoch": 1.16796875, "grad_norm": 0.002488090191036463, "learning_rate": 4.6488012907598146e-05, "loss": 3.5168559406884015e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00004, "step": 299, "tokens/total": 9732312, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 136705} +{"epoch": 1.171875, "grad_norm": 0.021835651248693466, "learning_rate": 4.620314165804964e-05, "loss": 0.00016595053602941334, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00017, "step": 300, "tokens/total": 9764624, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 137178} +{"epoch": 1.17578125, "grad_norm": 0.23852737247943878, "learning_rate": 4.591863648168407e-05, "loss": 0.0013417082373052835, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00134, "step": 301, "tokens/total": 9798368, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 137643} +{"epoch": 1.1796875, "grad_norm": 0.45403993129730225, "learning_rate": 4.5634509217923135e-05, "loss": 0.0077819800935685635, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00781, "step": 302, "tokens/total": 9830192, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 138078} +{"epoch": 1.18359375, "grad_norm": 0.01045912317931652, "learning_rate": 4.535077169046201e-05, "loss": 0.0001893745211418718, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00019, "step": 303, "tokens/total": 9863112, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 138515} +{"epoch": 1.1875, "grad_norm": 0.3627335727214813, "learning_rate": 4.506743570677743e-05, "loss": 0.003092047292739153, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.0031, "step": 304, "tokens/total": 9893192, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 138948} +{"epoch": 1.19140625, "grad_norm": 0.009004698134958744, "learning_rate": 4.478451305763618e-05, "loss": 0.00010037833999376744, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.0001, "step": 305, "tokens/total": 9925712, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 139408} +{"epoch": 1.1953125, "grad_norm": 0.09485701471567154, "learning_rate": 4.450201551660454e-05, "loss": 0.0009658998460508883, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00097, "step": 306, "tokens/total": 9955736, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 139840} +{"epoch": 1.19921875, "grad_norm": 0.09701340645551682, "learning_rate": 4.4219954839558276e-05, "loss": 0.0008117198012769222, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00081, "step": 307, "tokens/total": 9987160, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 140281} +{"epoch": 1.203125, "grad_norm": 0.855996310710907, "learning_rate": 4.393834276419352e-05, "loss": 0.0006068025249987841, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00061, "step": 308, "tokens/total": 10017360, "tokens/train_per_sec_per_gpu": 10.93, "tokens/trainable": 140776} +{"epoch": 1.20703125, "grad_norm": 0.6804371476173401, "learning_rate": 4.36571910095382e-05, "loss": 0.0008401582017540932, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00084, "step": 309, "tokens/total": 10052056, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 141228} +{"epoch": 1.2109375, "grad_norm": 0.006493649911135435, "learning_rate": 4.337651127546448e-05, "loss": 6.943676271475852e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00007, "step": 310, "tokens/total": 10085424, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 141679} +{"epoch": 1.21484375, "grad_norm": 0.458185613155365, "learning_rate": 4.3096315242201736e-05, "loss": 0.00293787126429379, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00294, "step": 311, "tokens/total": 10116824, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 142122} +{"epoch": 1.21875, "grad_norm": 0.013781899586319923, "learning_rate": 4.2816614569850635e-05, "loss": 6.796581146772951e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00007, "step": 312, "tokens/total": 10150376, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 142561} +{"epoch": 1.22265625, "grad_norm": 0.02732417918741703, "learning_rate": 4.2537420897897864e-05, "loss": 0.0002576496626716107, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00026, "step": 313, "tokens/total": 10184432, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 143046} +{"epoch": 1.2265625, "grad_norm": 0.01517858449369669, "learning_rate": 4.225874584473174e-05, "loss": 9.530907118460163e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.0001, "step": 314, "tokens/total": 10214848, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 143493} +{"epoch": 1.23046875, "grad_norm": 0.017115242779254913, "learning_rate": 4.19806010071587e-05, "loss": 0.00014301191549748182, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00014, "step": 315, "tokens/total": 10247952, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 143956} +{"epoch": 1.234375, "grad_norm": 0.02341507375240326, "learning_rate": 4.170299795992081e-05, "loss": 8.675218850839883e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00009, "step": 316, "tokens/total": 10280648, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 144411} +{"epoch": 1.23828125, "grad_norm": 0.014041000977158546, "learning_rate": 4.142594825521398e-05, "loss": 0.0001470299030188471, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00015, "step": 317, "tokens/total": 10317712, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 144892} +{"epoch": 1.2421875, "grad_norm": 0.0020891516469419003, "learning_rate": 4.114946342220728e-05, "loss": 4.195858127786778e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00004, "step": 318, "tokens/total": 10351632, "tokens/train_per_sec_per_gpu": 6.85, "tokens/trainable": 145339} +{"epoch": 1.24609375, "grad_norm": 0.12676872313022614, "learning_rate": 4.087355496656321e-05, "loss": 0.0003203489468432963, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00032, "step": 319, "tokens/total": 10383336, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 145811} +{"epoch": 1.25, "grad_norm": 0.6561354398727417, "learning_rate": 4.05982343699588e-05, "loss": 0.0015601222403347492, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00156, "step": 320, "tokens/total": 10418016, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 146314} +{"epoch": 1.25390625, "grad_norm": 0.0040266369469463825, "learning_rate": 4.0323513089607876e-05, "loss": 6.417079566745088e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00006, "step": 321, "tokens/total": 10451600, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 146781} +{"epoch": 1.2578125, "grad_norm": 0.03018275834619999, "learning_rate": 4.004940255778431e-05, "loss": 0.00014436017954722047, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00014, "step": 322, "tokens/total": 10487568, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 147225} +{"epoch": 1.26171875, "grad_norm": 0.005186598747968674, "learning_rate": 3.977591418134619e-05, "loss": 9.923068864736706e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.0001, "step": 323, "tokens/total": 10520976, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 147673} +{"epoch": 1.265625, "grad_norm": 0.007524341344833374, "learning_rate": 3.95030593412612e-05, "loss": 7.882552745286375e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00008, "step": 324, "tokens/total": 10552776, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 148103} +{"epoch": 1.26953125, "grad_norm": 0.014510632492601871, "learning_rate": 3.923084939213296e-05, "loss": 0.0001576091453898698, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00016, "step": 325, "tokens/total": 10582576, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 148535} +{"epoch": 1.2734375, "grad_norm": 0.003533915849402547, "learning_rate": 3.895929566172861e-05, "loss": 5.273747956380248e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00005, "step": 326, "tokens/total": 10615600, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 148999} +{"epoch": 1.27734375, "grad_norm": 0.08147163689136505, "learning_rate": 3.868840945050728e-05, "loss": 0.00034175452310591936, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00034, "step": 327, "tokens/total": 10650688, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 149431} +{"epoch": 1.28125, "grad_norm": 0.00617097457870841, "learning_rate": 3.841820203114995e-05, "loss": 4.380283280625008e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00004, "step": 328, "tokens/total": 10681816, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 149886} +{"epoch": 1.28515625, "grad_norm": 0.05087108165025711, "learning_rate": 3.814868464809027e-05, "loss": 0.0005463320994749665, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00055, "step": 329, "tokens/total": 10711728, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 150288} +{"epoch": 1.2890625, "grad_norm": 0.061403680592775345, "learning_rate": 3.787986851704667e-05, "loss": 0.0004740164731629193, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00047, "step": 330, "tokens/total": 10744952, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 150733} +{"epoch": 1.29296875, "grad_norm": 0.6325178146362305, "learning_rate": 3.7611764824555654e-05, "loss": 0.0005404020776040852, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00054, "step": 331, "tokens/total": 10776192, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 151207} +{"epoch": 1.296875, "grad_norm": 0.0035310632083564997, "learning_rate": 3.734438472750619e-05, "loss": 3.4678017982514575e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00003, "step": 332, "tokens/total": 10813336, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 151694} +{"epoch": 1.30078125, "grad_norm": 0.09093944728374481, "learning_rate": 3.707773935267552e-05, "loss": 0.00016556825721636415, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00017, "step": 333, "tokens/total": 10847808, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 152188} +{"epoch": 1.3046875, "grad_norm": 0.0018124983180314302, "learning_rate": 3.68118397962661e-05, "loss": 2.1021831344114617e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00002, "step": 334, "tokens/total": 10879896, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 152596} +{"epoch": 1.30859375, "grad_norm": 0.025279050692915916, "learning_rate": 3.654669712344384e-05, "loss": 0.0001712775119813159, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00017, "step": 335, "tokens/total": 10908976, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 153052} +{"epoch": 1.3125, "grad_norm": 0.008354760706424713, "learning_rate": 3.628232236787763e-05, "loss": 6.206766556715593e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00006, "step": 336, "tokens/total": 10943152, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 153491} +{"epoch": 1.31640625, "grad_norm": 0.01810077764093876, "learning_rate": 3.6018726531280144e-05, "loss": 0.00014194620598573238, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00014, "step": 337, "tokens/total": 10978184, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 153983} +{"epoch": 1.3203125, "grad_norm": 0.024006016552448273, "learning_rate": 3.575592058295017e-05, "loss": 0.00017549010226503015, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00018, "step": 338, "tokens/total": 11010056, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 154464} +{"epoch": 1.32421875, "grad_norm": 0.0036396586801856756, "learning_rate": 3.549391545931585e-05, "loss": 1.8858074326999485e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00002, "step": 339, "tokens/total": 11040968, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 154924} +{"epoch": 1.328125, "grad_norm": 0.0005936053348705173, "learning_rate": 3.5232722063479914e-05, "loss": 9.002640581456944e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00001, "step": 340, "tokens/total": 11073424, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 155373} +{"epoch": 1.33203125, "grad_norm": 0.003551068250089884, "learning_rate": 3.49723512647657e-05, "loss": 5.930437328061089e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00006, "step": 341, "tokens/total": 11109224, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 155873} +{"epoch": 1.3359375, "grad_norm": 0.0016364288749173284, "learning_rate": 3.471281389826491e-05, "loss": 1.0073921657749452e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00001, "step": 342, "tokens/total": 11139856, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 156278} +{"epoch": 1.33984375, "grad_norm": 0.003502934705466032, "learning_rate": 3.4454120764386764e-05, "loss": 3.395946623641066e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00003, "step": 343, "tokens/total": 11175224, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 156733} +{"epoch": 1.34375, "grad_norm": 0.02962476760149002, "learning_rate": 3.4196282628408526e-05, "loss": 0.00010490968270460144, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.0001, "step": 344, "tokens/total": 11207112, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 157203} +{"epoch": 1.34765625, "grad_norm": 0.23435437679290771, "learning_rate": 3.3939310220027456e-05, "loss": 0.0003769106697291136, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00038, "step": 345, "tokens/total": 11240432, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 157707} +{"epoch": 1.3515625, "grad_norm": 0.0036909820046275854, "learning_rate": 3.3683214232914404e-05, "loss": 3.0076136681600474e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00003, "step": 346, "tokens/total": 11274584, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 158180} +{"epoch": 1.35546875, "grad_norm": 0.0363287627696991, "learning_rate": 3.342800532426873e-05, "loss": 0.00021455106616485864, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00021, "step": 347, "tokens/total": 11307376, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 158646} +{"epoch": 1.359375, "grad_norm": 0.08291535079479218, "learning_rate": 3.317369411437484e-05, "loss": 0.0004726297629531473, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00047, "step": 348, "tokens/total": 11340640, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 159115} +{"epoch": 1.36328125, "grad_norm": 0.052187204360961914, "learning_rate": 3.292029118616024e-05, "loss": 0.0001420476328348741, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00014, "step": 349, "tokens/total": 11371632, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 159538} +{"epoch": 1.3671875, "grad_norm": 0.2290094941854477, "learning_rate": 3.266780708475511e-05, "loss": 0.0047728135250508785, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00478, "step": 350, "tokens/total": 11404248, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 160016} +{"epoch": 1.37109375, "grad_norm": 0.005166149698197842, "learning_rate": 3.241625231705354e-05, "loss": 3.246923006372526e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00003, "step": 351, "tokens/total": 11439552, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 160475} +{"epoch": 1.375, "grad_norm": 0.0005873373593203723, "learning_rate": 3.216563735127618e-05, "loss": 8.608681127952877e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00001, "step": 352, "tokens/total": 11470944, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 160952} +{"epoch": 1.37890625, "grad_norm": 0.011665266007184982, "learning_rate": 3.191597261653475e-05, "loss": 7.231803465401754e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00007, "step": 353, "tokens/total": 11506304, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 161420} +{"epoch": 1.3828125, "grad_norm": 0.0030789682641625404, "learning_rate": 3.166726850239794e-05, "loss": 3.332766209496185e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00003, "step": 354, "tokens/total": 11535424, "tokens/train_per_sec_per_gpu": 10.78, "tokens/trainable": 161868} +{"epoch": 1.38671875, "grad_norm": 0.011923485435545444, "learning_rate": 3.141953535845912e-05, "loss": 4.899367195321247e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00005, "step": 355, "tokens/total": 11565176, "tokens/train_per_sec_per_gpu": 10.07, "tokens/trainable": 162318} +{"epoch": 1.390625, "grad_norm": 0.0022940959315747023, "learning_rate": 3.11727834939056e-05, "loss": 2.0011584638268687e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00002, "step": 356, "tokens/total": 11595528, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 162789} +{"epoch": 1.39453125, "grad_norm": 0.03261147439479828, "learning_rate": 3.092702317708967e-05, "loss": 0.00015014265954960138, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00015, "step": 357, "tokens/total": 11627248, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 163254} +{"epoch": 1.3984375, "grad_norm": 0.0002474926004651934, "learning_rate": 3.0682264635101276e-05, "loss": 5.9308172239980195e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00001, "step": 358, "tokens/total": 11659440, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 163715} +{"epoch": 1.40234375, "grad_norm": 0.009927228093147278, "learning_rate": 3.0438518053342407e-05, "loss": 4.6107736125122756e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00005, "step": 359, "tokens/total": 11693288, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 164197} +{"epoch": 1.40625, "grad_norm": 0.0032429713755846024, "learning_rate": 3.0195793575103266e-05, "loss": 2.281252818647772e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00002, "step": 360, "tokens/total": 11730888, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 164661} +{"epoch": 1.41015625, "grad_norm": 0.10182265192270279, "learning_rate": 2.9954101301140146e-05, "loss": 0.0005555460229516029, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00056, "step": 361, "tokens/total": 11765656, "tokens/train_per_sec_per_gpu": 6.79, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.003299278672784567, "learning_rate": 2.9713451289255123e-05, "loss": 4.6322242269525304e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00005, "step": 362, "tokens/total": 11793456, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 165552} +{"epoch": 1.41796875, "grad_norm": 0.0003635539032984525, "learning_rate": 2.9473853553877484e-05, "loss": 6.941691935935523e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00001, "step": 363, "tokens/total": 11826280, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 166026} +{"epoch": 1.421875, "grad_norm": 0.0037349441554397345, "learning_rate": 2.9235318065647e-05, "loss": 1.6624017007416114e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00002, "step": 364, "tokens/total": 11863552, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 166486} +{"epoch": 1.42578125, "grad_norm": 0.0027791564352810383, "learning_rate": 2.8997854750998964e-05, "loss": 2.1919295249972492e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00002, "step": 365, "tokens/total": 11898616, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 166959} +{"epoch": 1.4296875, "grad_norm": 0.00212305155582726, "learning_rate": 2.8761473491751258e-05, "loss": 1.85354620043654e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00002, "step": 366, "tokens/total": 11930728, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 167394} +{"epoch": 1.43359375, "grad_norm": 0.010948137380182743, "learning_rate": 2.8526184124692883e-05, "loss": 2.444460551487282e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00002, "step": 367, "tokens/total": 11964976, "tokens/train_per_sec_per_gpu": 6.29, "tokens/trainable": 167832} +{"epoch": 1.4375, "grad_norm": 0.12363535165786743, "learning_rate": 2.829199644117484e-05, "loss": 0.00023618899285793304, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00024, "step": 368, "tokens/total": 11996696, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 168242} +{"epoch": 1.44140625, "grad_norm": 0.40875089168548584, "learning_rate": 2.8058920186702553e-05, "loss": 0.00240929052233696, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00241, "step": 369, "tokens/total": 12033432, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 168715} +{"epoch": 1.4453125, "grad_norm": 0.00919466745108366, "learning_rate": 2.782696506053033e-05, "loss": 4.011232158518396e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00004, "step": 370, "tokens/total": 12067616, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 169147} +{"epoch": 1.44921875, "grad_norm": 1.1478748321533203, "learning_rate": 2.7596140715257824e-05, "loss": 0.011827483773231506, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.0119, "step": 371, "tokens/total": 12099008, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 169628} +{"epoch": 1.453125, "grad_norm": 0.0017901661340147257, "learning_rate": 2.7366456756428184e-05, "loss": 1.1239087143621873e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00001, "step": 372, "tokens/total": 12131304, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 170085} +{"epoch": 1.45703125, "grad_norm": 0.0007103498792275786, "learning_rate": 2.7137922742128486e-05, "loss": 9.91171873465646e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00001, "step": 373, "tokens/total": 12164104, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 170584} +{"epoch": 1.4609375, "grad_norm": 0.00940308254212141, "learning_rate": 2.691054818259188e-05, "loss": 5.692222111974843e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00006, "step": 374, "tokens/total": 12197776, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 171058} +{"epoch": 1.46484375, "grad_norm": 0.003377815242856741, "learning_rate": 2.6684342539801933e-05, "loss": 3.215713513782248e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00003, "step": 375, "tokens/total": 12228064, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 171518} +{"epoch": 1.46875, "grad_norm": 0.0011857234640046954, "learning_rate": 2.645931522709877e-05, "loss": 1.631988925510086e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00002, "step": 376, "tokens/total": 12261688, "tokens/train_per_sec_per_gpu": 10.56, "tokens/trainable": 172007} +{"epoch": 1.47265625, "grad_norm": 0.004236388020217419, "learning_rate": 2.6235475608787365e-05, "loss": 4.0459093725075945e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00004, "step": 377, "tokens/total": 12294240, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 172468} +{"epoch": 1.4765625, "grad_norm": 0.024118896573781967, "learning_rate": 2.6012832999747916e-05, "loss": 0.00012062511086696759, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00012, "step": 378, "tokens/total": 12324528, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 172923} +{"epoch": 1.48046875, "grad_norm": 0.7160465717315674, "learning_rate": 2.579139666504821e-05, "loss": 0.007811105344444513, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00784, "step": 379, "tokens/total": 12360288, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 173370} +{"epoch": 1.484375, "grad_norm": 0.3162981867790222, "learning_rate": 2.557117581955798e-05, "loss": 0.002466406673192978, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00247, "step": 380, "tokens/total": 12391216, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 173837} +{"epoch": 1.48828125, "grad_norm": 0.0025110577698796988, "learning_rate": 2.5352179627565532e-05, "loss": 2.0999603293603286e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00002, "step": 381, "tokens/total": 12423624, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 174294} +{"epoch": 1.4921875, "grad_norm": 0.11332306265830994, "learning_rate": 2.5134417202396277e-05, "loss": 0.00016077446343842894, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00016, "step": 382, "tokens/total": 12456032, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 174701} +{"epoch": 1.49609375, "grad_norm": 0.660546600818634, "learning_rate": 2.491789760603361e-05, "loss": 0.01007828675210476, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.01013, "step": 383, "tokens/total": 12489336, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 175131} +{"epoch": 1.5, "grad_norm": 0.09495681524276733, "learning_rate": 2.4702629848741764e-05, "loss": 0.00031842905445955694, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00032, "step": 384, "tokens/total": 12521944, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 175586} +{"epoch": 1.50390625, "grad_norm": 0.18169333040714264, "learning_rate": 2.4488622888690785e-05, "loss": 0.00104867247864604, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00105, "step": 385, "tokens/total": 12554024, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 176026} +{"epoch": 1.5078125, "grad_norm": 0.03114008530974388, "learning_rate": 2.427588563158384e-05, "loss": 0.00018476576951798052, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00018, "step": 386, "tokens/total": 12587104, "tokens/train_per_sec_per_gpu": 7.04, "tokens/trainable": 176512} +{"epoch": 1.51171875, "grad_norm": 0.005054978188127279, "learning_rate": 2.406442693028651e-05, "loss": 2.7414833311922848e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00003, "step": 387, "tokens/total": 12617456, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 176943} +{"epoch": 1.515625, "grad_norm": 0.003451877972111106, "learning_rate": 2.3854255584458547e-05, "loss": 2.236000364064239e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00002, "step": 388, "tokens/total": 12652696, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 177370} +{"epoch": 1.51953125, "grad_norm": 0.007548264227807522, "learning_rate": 2.3645380340187508e-05, "loss": 2.5673742129583843e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00003, "step": 389, "tokens/total": 12685792, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 177851} +{"epoch": 1.5234375, "grad_norm": 0.09520450234413147, "learning_rate": 2.3437809889624914e-05, "loss": 0.0002670914400368929, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00027, "step": 390, "tokens/total": 12716120, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 178310} +{"epoch": 1.52734375, "grad_norm": 0.015687691047787666, "learning_rate": 2.3231552870624487e-05, "loss": 9.582663915352896e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.0001, "step": 391, "tokens/total": 12746216, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 178736} +{"epoch": 1.53125, "grad_norm": 0.5641492605209351, "learning_rate": 2.3026617866382657e-05, "loss": 0.00508877681568265, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.0051, "step": 392, "tokens/total": 12778592, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 179233} +{"epoch": 1.53515625, "grad_norm": 0.008262171410024166, "learning_rate": 2.2823013405081507e-05, "loss": 6.0816768382210284e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00006, "step": 393, "tokens/total": 12812008, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 179730} +{"epoch": 1.5390625, "grad_norm": 0.0017360053025186062, "learning_rate": 2.2620747959533722e-05, "loss": 2.11313981708372e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00002, "step": 394, "tokens/total": 12844264, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 180227} +{"epoch": 1.54296875, "grad_norm": 0.03889116644859314, "learning_rate": 2.2419829946830123e-05, "loss": 0.00021506576740648597, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00022, "step": 395, "tokens/total": 12878328, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 180687} +{"epoch": 1.546875, "grad_norm": 0.12844830751419067, "learning_rate": 2.2220267727989325e-05, "loss": 0.0009910304797813296, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00099, "step": 396, "tokens/total": 12907912, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 181141} +{"epoch": 1.55078125, "grad_norm": 0.05962744355201721, "learning_rate": 2.202206960760984e-05, "loss": 0.0003158766485285014, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00032, "step": 397, "tokens/total": 12941968, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 181648} +{"epoch": 1.5546875, "grad_norm": 0.2122466266155243, "learning_rate": 2.182524383352446e-05, "loss": 0.001558191841468215, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00156, "step": 398, "tokens/total": 12975240, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 182109} +{"epoch": 1.55859375, "grad_norm": 0.3243716359138489, "learning_rate": 2.1629798596457056e-05, "loss": 0.006867108400911093, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00689, "step": 399, "tokens/total": 13006912, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 182559} +{"epoch": 1.5625, "grad_norm": 0.020060604438185692, "learning_rate": 2.1435742029681725e-05, "loss": 0.00013915861200075597, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00014, "step": 400, "tokens/total": 13037640, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 182985} +{"epoch": 1.56640625, "grad_norm": 0.06307636201381683, "learning_rate": 2.124308220868431e-05, "loss": 5.3772331739310175e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00005, "step": 401, "tokens/total": 13070608, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 183440} +{"epoch": 1.5703125, "grad_norm": 0.01878008060157299, "learning_rate": 2.105182715082638e-05, "loss": 0.00018721552623901516, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00019, "step": 402, "tokens/total": 13101784, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 183917} +{"epoch": 1.57421875, "grad_norm": 0.012431113049387932, "learning_rate": 2.0861984815011552e-05, "loss": 8.974657976068556e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00009, "step": 403, "tokens/total": 13133528, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 184348} +{"epoch": 1.578125, "grad_norm": 0.03223200514912605, "learning_rate": 2.0673563101354323e-05, "loss": 0.00023369473638013005, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00023, "step": 404, "tokens/total": 13167088, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 184786} +{"epoch": 1.58203125, "grad_norm": 0.005490154959261417, "learning_rate": 2.0486569850851317e-05, "loss": 3.473047399893403e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00003, "step": 405, "tokens/total": 13200392, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 185249} +{"epoch": 1.5859375, "grad_norm": 0.04321201518177986, "learning_rate": 2.0301012845054956e-05, "loss": 0.00017850182484835386, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00018, "step": 406, "tokens/total": 13235488, "tokens/train_per_sec_per_gpu": 6.75, "tokens/trainable": 185680} +{"epoch": 1.58984375, "grad_norm": 0.006028336007148027, "learning_rate": 2.011689980574966e-05, "loss": 5.1017559599131346e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00005, "step": 407, "tokens/total": 13269416, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 186100} +{"epoch": 1.59375, "grad_norm": 0.04493558034300804, "learning_rate": 1.993423839463052e-05, "loss": 0.0002618888975121081, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00026, "step": 408, "tokens/total": 13301360, "tokens/train_per_sec_per_gpu": 10.23, "tokens/trainable": 186565} +{"epoch": 1.59765625, "grad_norm": 0.02385113202035427, "learning_rate": 1.975303621298445e-05, "loss": 0.0001071925216820091, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00011, "step": 409, "tokens/total": 13336392, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 187017} +{"epoch": 1.6015625, "grad_norm": 0.15817032754421234, "learning_rate": 1.957330080137385e-05, "loss": 0.0004831683181691915, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00048, "step": 410, "tokens/total": 13366072, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 187468} +{"epoch": 1.60546875, "grad_norm": 0.08318479359149933, "learning_rate": 1.9395039639322864e-05, "loss": 0.00030938402051106095, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00031, "step": 411, "tokens/total": 13400832, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 187961} +{"epoch": 1.609375, "grad_norm": 0.003513800213113427, "learning_rate": 1.9218260145006073e-05, "loss": 2.2757649276172742e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00002, "step": 412, "tokens/total": 13434920, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 188447} +{"epoch": 1.61328125, "grad_norm": 0.0011997068068012595, "learning_rate": 1.904296967493982e-05, "loss": 1.5093351976247504e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00002, "step": 413, "tokens/total": 13466512, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 188903} +{"epoch": 1.6171875, "grad_norm": 0.01997177116572857, "learning_rate": 1.8869175523676064e-05, "loss": 0.00011196920968359336, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00011, "step": 414, "tokens/total": 13500352, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 189398} +{"epoch": 1.62109375, "grad_norm": 0.004133118316531181, "learning_rate": 1.869688492349885e-05, "loss": 5.907137528993189e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00006, "step": 415, "tokens/total": 13532448, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 189850} +{"epoch": 1.625, "grad_norm": 0.20153360068798065, "learning_rate": 1.85261050441233e-05, "loss": 0.0018173855496570468, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00182, "step": 416, "tokens/total": 13565648, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 190306} +{"epoch": 1.62890625, "grad_norm": 0.11040833592414856, "learning_rate": 1.8356842992397304e-05, "loss": 0.0006185292731970549, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00062, "step": 417, "tokens/total": 13597600, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 190746} +{"epoch": 1.6328125, "grad_norm": 0.3520786166191101, "learning_rate": 1.8189105812005714e-05, "loss": 0.0027773031033575535, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00278, "step": 418, "tokens/total": 13630456, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 191156} +{"epoch": 1.63671875, "grad_norm": 0.002174732042476535, "learning_rate": 1.802290048317732e-05, "loss": 3.018094867002219e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00003, "step": 419, "tokens/total": 13660896, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 191573} +{"epoch": 1.640625, "grad_norm": 0.002735738642513752, "learning_rate": 1.785823392239424e-05, "loss": 3.992966958321631e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00004, "step": 420, "tokens/total": 13693176, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 192014} +{"epoch": 1.64453125, "grad_norm": 0.028530655428767204, "learning_rate": 1.7695112982104225e-05, "loss": 0.00022627718863077462, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00023, "step": 421, "tokens/total": 13725400, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 192453} +{"epoch": 1.6484375, "grad_norm": 0.0013137331698089838, "learning_rate": 1.7533544450435433e-05, "loss": 2.2991767764324322e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00002, "step": 422, "tokens/total": 13752752, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 192886} +{"epoch": 1.65234375, "grad_norm": 0.0032316979486495256, "learning_rate": 1.7373535050913946e-05, "loss": 1.7177480913233012e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00002, "step": 423, "tokens/total": 13786136, "tokens/train_per_sec_per_gpu": 7.04, "tokens/trainable": 193323} +{"epoch": 1.65625, "grad_norm": 0.004849154036492109, "learning_rate": 1.721509144218405e-05, "loss": 8.19139095256105e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00008, "step": 424, "tokens/total": 13815904, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 193753} +{"epoch": 1.66015625, "grad_norm": 0.16752362251281738, "learning_rate": 1.705822021773101e-05, "loss": 0.0003181381616741419, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00032, "step": 425, "tokens/total": 13847976, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 194240} +{"epoch": 1.6640625, "grad_norm": 0.32283923029899597, "learning_rate": 1.69029279056068e-05, "loss": 0.0016055782325565815, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00161, "step": 426, "tokens/total": 13883200, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 194680} +{"epoch": 1.66796875, "grad_norm": 0.0030126883648335934, "learning_rate": 1.6749220968158415e-05, "loss": 2.890779433073476e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00003, "step": 427, "tokens/total": 13916080, "tokens/train_per_sec_per_gpu": 10.18, "tokens/trainable": 195186} +{"epoch": 1.671875, "grad_norm": 0.07902218401432037, "learning_rate": 1.659710580175893e-05, "loss": 0.0006663288804702461, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00067, "step": 428, "tokens/total": 13946448, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 195693} +{"epoch": 1.67578125, "grad_norm": 0.048067666590213776, "learning_rate": 1.644658873654133e-05, "loss": 5.466732181957923e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00005, "step": 429, "tokens/total": 13975696, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 196130} +{"epoch": 1.6796875, "grad_norm": 0.0045107691548764706, "learning_rate": 1.629767603613508e-05, "loss": 2.1328043658286333e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00002, "step": 430, "tokens/total": 14002616, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 196530} +{"epoch": 1.68359375, "grad_norm": 0.0019261750858277082, "learning_rate": 1.615037389740547e-05, "loss": 2.8321192075964063e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00003, "step": 431, "tokens/total": 14034848, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 196974} +{"epoch": 1.6875, "grad_norm": 0.0026274900883436203, "learning_rate": 1.600468845019576e-05, "loss": 3.3208416425623e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00003, "step": 432, "tokens/total": 14069032, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 197454} +{"epoch": 1.69140625, "grad_norm": 0.00032164418371394277, "learning_rate": 1.5860625757072092e-05, "loss": 6.7511382439988665e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00001, "step": 433, "tokens/total": 14101296, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 197902} +{"epoch": 1.6953125, "grad_norm": 0.004245315678417683, "learning_rate": 1.571819181307116e-05, "loss": 2.7964839318883605e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00003, "step": 434, "tokens/total": 14135880, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 198374} +{"epoch": 1.69921875, "grad_norm": 0.005515324417501688, "learning_rate": 1.557739254545075e-05, "loss": 3.255354386055842e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00003, "step": 435, "tokens/total": 14166040, "tokens/train_per_sec_per_gpu": 10.45, "tokens/trainable": 198846} +{"epoch": 1.703125, "grad_norm": 0.005131261423230171, "learning_rate": 1.543823381344311e-05, "loss": 5.010070890421048e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00005, "step": 436, "tokens/total": 14196032, "tokens/train_per_sec_per_gpu": 10.57, "tokens/trainable": 199320} +{"epoch": 1.70703125, "grad_norm": 0.014392673969268799, "learning_rate": 1.5300721408011114e-05, "loss": 7.23689008736983e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00007, "step": 437, "tokens/total": 14228008, "tokens/train_per_sec_per_gpu": 10.05, "tokens/trainable": 199777} +{"epoch": 1.7109375, "grad_norm": 0.0060644689947366714, "learning_rate": 1.5164861051607254e-05, "loss": 3.9961498259799555e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.76, "memory/max_allocated (GiB)": 65.76, "ppl": 1.00004, "step": 438, "tokens/total": 14261744, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 200194} +{"epoch": 1.71484375, "grad_norm": 0.0007904747035354376, "learning_rate": 1.5030658397935521e-05, "loss": 7.970810656843241e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00001, "step": 439, "tokens/total": 14295208, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 200663} +{"epoch": 1.71875, "grad_norm": 0.008237929083406925, "learning_rate": 1.4898119031716104e-05, "loss": 5.8066972997039557e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00006, "step": 440, "tokens/total": 14327264, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 201121} +{"epoch": 1.72265625, "grad_norm": 0.1512545496225357, "learning_rate": 1.476724846845306e-05, "loss": 0.0016355165280401707, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00164, "step": 441, "tokens/total": 14363088, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 201598} +{"epoch": 1.7265625, "grad_norm": 0.011590167880058289, "learning_rate": 1.463805215420471e-05, "loss": 9.73014030023478e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.0001, "step": 442, "tokens/total": 14396088, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 202068} +{"epoch": 1.73046875, "grad_norm": 0.05370739474892616, "learning_rate": 1.451053546535705e-05, "loss": 0.0001847445237217471, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00018, "step": 443, "tokens/total": 14429232, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 202524} +{"epoch": 1.734375, "grad_norm": 0.006353891454637051, "learning_rate": 1.438470370840001e-05, "loss": 3.344373908475973e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00003, "step": 444, "tokens/total": 14463264, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 202996} +{"epoch": 1.73828125, "grad_norm": 0.009142174385488033, "learning_rate": 1.4260562119706606e-05, "loss": 5.845843406859785e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00006, "step": 445, "tokens/total": 14493552, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 203470} +{"epoch": 1.7421875, "grad_norm": 0.009692071005702019, "learning_rate": 1.413811586531508e-05, "loss": 4.4296819396549836e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00004, "step": 446, "tokens/total": 14525240, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 203919} +{"epoch": 1.74609375, "grad_norm": 0.1524532288312912, "learning_rate": 1.4017370040713884e-05, "loss": 0.0005676134023815393, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00057, "step": 447, "tokens/total": 14559560, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 204376} +{"epoch": 1.75, "grad_norm": 0.05978335067629814, "learning_rate": 1.3898329670629645e-05, "loss": 0.0002492911007720977, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00025, "step": 448, "tokens/total": 14594160, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 204821} +{"epoch": 1.75390625, "grad_norm": 0.0009353029890917242, "learning_rate": 1.3780999708818058e-05, "loss": 9.514958037470933e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00001, "step": 449, "tokens/total": 14624224, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 205271} +{"epoch": 1.7578125, "grad_norm": 0.003336181864142418, "learning_rate": 1.3665385037857758e-05, "loss": 3.6138670111540705e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00004, "step": 450, "tokens/total": 14654984, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 205723} +{"epoch": 1.76171875, "grad_norm": 0.012936298735439777, "learning_rate": 1.3551490468947126e-05, "loss": 8.059953688643873e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00008, "step": 451, "tokens/total": 14685440, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 206169} +{"epoch": 1.765625, "grad_norm": 0.0019805387128144503, "learning_rate": 1.3439320741704075e-05, "loss": 2.7883514121640474e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00003, "step": 452, "tokens/total": 14716120, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 206592} +{"epoch": 1.76953125, "grad_norm": 0.005173974670469761, "learning_rate": 1.3328880523968808e-05, "loss": 4.193321365164593e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00004, "step": 453, "tokens/total": 14748424, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 207055} +{"epoch": 1.7734375, "grad_norm": 0.010267865844070911, "learning_rate": 1.3220174411609587e-05, "loss": 4.185225770925172e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00004, "step": 454, "tokens/total": 14781256, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 207494} +{"epoch": 1.77734375, "grad_norm": 0.03472502529621124, "learning_rate": 1.3113206928331471e-05, "loss": 0.0002562986337579787, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00026, "step": 455, "tokens/total": 14809552, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 207956} +{"epoch": 1.78125, "grad_norm": 0.0005914925714023411, "learning_rate": 1.300798252548806e-05, "loss": 1.0114898941537831e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00001, "step": 456, "tokens/total": 14841808, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 208408} +{"epoch": 1.78515625, "grad_norm": 0.0011057937517762184, "learning_rate": 1.2904505581896265e-05, "loss": 1.2550231076602358e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00001, "step": 457, "tokens/total": 14875896, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 208889} +{"epoch": 1.7890625, "grad_norm": 0.010306614451110363, "learning_rate": 1.2802780403654082e-05, "loss": 0.00011160357826156542, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00011, "step": 458, "tokens/total": 14908424, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 209336} +{"epoch": 1.79296875, "grad_norm": 0.006658047903329134, "learning_rate": 1.2702811223961408e-05, "loss": 5.8983598137274384e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00006, "step": 459, "tokens/total": 14939168, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 209797} +{"epoch": 1.796875, "grad_norm": 0.018350161612033844, "learning_rate": 1.2604602202943861e-05, "loss": 6.97664509061724e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00007, "step": 460, "tokens/total": 14969872, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 210240} +{"epoch": 1.80078125, "grad_norm": 0.0035923649556934834, "learning_rate": 1.2508157427479686e-05, "loss": 3.678346183733083e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00004, "step": 461, "tokens/total": 15001472, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 210702} +{"epoch": 1.8046875, "grad_norm": 0.0027283490635454655, "learning_rate": 1.2413480911029655e-05, "loss": 2.2118823835626245e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00002, "step": 462, "tokens/total": 15030592, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 211140} +{"epoch": 1.80859375, "grad_norm": 0.0005858240183442831, "learning_rate": 1.2320576593470082e-05, "loss": 9.356983355246484e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00001, "step": 463, "tokens/total": 15063648, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 211575} +{"epoch": 1.8125, "grad_norm": 0.0023060108069330454, "learning_rate": 1.2229448340928828e-05, "loss": 2.7395446522859856e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00003, "step": 464, "tokens/total": 15098248, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 212083} +{"epoch": 1.81640625, "grad_norm": 0.0020516819786280394, "learning_rate": 1.2140099945624458e-05, "loss": 2.087128632410895e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00002, "step": 465, "tokens/total": 15133216, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 212558} +{"epoch": 1.8203125, "grad_norm": 0.004668538458645344, "learning_rate": 1.205253512570841e-05, "loss": 2.473805216141045e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00002, "step": 466, "tokens/total": 15165416, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 213014} +{"epoch": 1.82421875, "grad_norm": 0.009027127176523209, "learning_rate": 1.1966757525110255e-05, "loss": 6.893704266985878e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00007, "step": 467, "tokens/total": 15193688, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 213465} +{"epoch": 1.828125, "grad_norm": 0.05204543098807335, "learning_rate": 1.1882770713386095e-05, "loss": 0.00026316000730730593, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00026, "step": 468, "tokens/total": 15228600, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 213940} +{"epoch": 1.83203125, "grad_norm": 0.03502952307462692, "learning_rate": 1.180057818556998e-05, "loss": 0.00020625180331990123, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00021, "step": 469, "tokens/total": 15262192, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 214415} +{"epoch": 1.8359375, "grad_norm": 0.019239839166402817, "learning_rate": 1.1720183362028494e-05, "loss": 0.0001398929744027555, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00014, "step": 470, "tokens/total": 15294920, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 214880} +{"epoch": 1.83984375, "grad_norm": 0.033325787633657455, "learning_rate": 1.1641589588318387e-05, "loss": 0.00013238389510661364, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00013, "step": 471, "tokens/total": 15327432, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 215334} +{"epoch": 1.84375, "grad_norm": 0.0003643939853645861, "learning_rate": 1.1564800135047418e-05, "loss": 5.375691216613632e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00001, "step": 472, "tokens/total": 15359928, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 215822} +{"epoch": 1.84765625, "grad_norm": 0.0012312742182984948, "learning_rate": 1.148981819773816e-05, "loss": 8.043251909839455e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00001, "step": 473, "tokens/total": 15391400, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 216239} +{"epoch": 1.8515625, "grad_norm": 0.002942553022876382, "learning_rate": 1.1416646896695086e-05, "loss": 2.0749521354446188e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00002, "step": 474, "tokens/total": 15424144, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 216691} +{"epoch": 1.85546875, "grad_norm": 0.027531689032912254, "learning_rate": 1.1345289276874717e-05, "loss": 0.00016950858116615564, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00017, "step": 475, "tokens/total": 15457680, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 217162} +{"epoch": 1.859375, "grad_norm": 0.11602059006690979, "learning_rate": 1.1275748307758873e-05, "loss": 0.0006824492011219263, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00068, "step": 476, "tokens/total": 15492056, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 217638} +{"epoch": 1.86328125, "grad_norm": 0.0021319768857210875, "learning_rate": 1.1208026883231147e-05, "loss": 1.8682856534724124e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00002, "step": 477, "tokens/total": 15521544, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 218053} +{"epoch": 1.8671875, "grad_norm": 0.0024446160532534122, "learning_rate": 1.1142127821456433e-05, "loss": 2.9177779651945457e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00003, "step": 478, "tokens/total": 15553320, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 218503} +{"epoch": 1.87109375, "grad_norm": 0.016486141830682755, "learning_rate": 1.1078053864763674e-05, "loss": 8.598638669354841e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00009, "step": 479, "tokens/total": 15586208, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 218964} +{"epoch": 1.875, "grad_norm": 0.16331706941127777, "learning_rate": 1.1015807679531756e-05, "loss": 0.0006117009907029569, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00061, "step": 480, "tokens/total": 15616432, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 219412} +{"epoch": 1.87890625, "grad_norm": 0.06384844332933426, "learning_rate": 1.0955391856078528e-05, "loss": 0.00036487056058831513, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00036, "step": 481, "tokens/total": 15650056, "tokens/train_per_sec_per_gpu": 10.13, "tokens/trainable": 219886} +{"epoch": 1.8828125, "grad_norm": 0.004974262788891792, "learning_rate": 1.0896808908553007e-05, "loss": 4.5793924073223025e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00005, "step": 482, "tokens/total": 15680336, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 220343} +{"epoch": 1.88671875, "grad_norm": 0.018407387658953667, "learning_rate": 1.0840061274830763e-05, "loss": 9.25138738239184e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00009, "step": 483, "tokens/total": 15711552, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 220825} +{"epoch": 1.890625, "grad_norm": 0.0015788625460118055, "learning_rate": 1.0785151316412473e-05, "loss": 1.715456528472714e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00002, "step": 484, "tokens/total": 15744664, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 221282} +{"epoch": 1.89453125, "grad_norm": 0.0021038351114839315, "learning_rate": 1.0732081318325639e-05, "loss": 1.8857628674595617e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00002, "step": 485, "tokens/total": 15782800, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 221780} +{"epoch": 1.8984375, "grad_norm": 0.05259989574551582, "learning_rate": 1.0680853489029501e-05, "loss": 0.0001782536564860493, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00018, "step": 486, "tokens/total": 15817368, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 222210} +{"epoch": 1.90234375, "grad_norm": 0.00947787705808878, "learning_rate": 1.0631469960323152e-05, "loss": 2.1353929696488194e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00002, "step": 487, "tokens/total": 15850408, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 222646} +{"epoch": 1.90625, "grad_norm": 0.03659040853381157, "learning_rate": 1.0583932787256783e-05, "loss": 0.0001716020778985694, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00017, "step": 488, "tokens/total": 15881448, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 223066} +{"epoch": 1.91015625, "grad_norm": 0.00018162514606956393, "learning_rate": 1.0538243948046206e-05, "loss": 3.9196265788632445e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.0, "step": 489, "tokens/total": 15913736, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 223512} +{"epoch": 1.9140625, "grad_norm": 0.00030590730602853, "learning_rate": 1.0494405343990523e-05, "loss": 6.158100404718425e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00001, "step": 490, "tokens/total": 15944568, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 223956} +{"epoch": 1.91796875, "grad_norm": 0.003836406394839287, "learning_rate": 1.0452418799392985e-05, "loss": 4.0793420339468867e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00004, "step": 491, "tokens/total": 15976352, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 224416} +{"epoch": 1.921875, "grad_norm": 0.0003709495940711349, "learning_rate": 1.0412286061485102e-05, "loss": 5.9611656979541294e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00001, "step": 492, "tokens/total": 16010392, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 224883} +{"epoch": 1.92578125, "grad_norm": 0.0037100266199558973, "learning_rate": 1.03740088003539e-05, "loss": 2.6904333935817704e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00003, "step": 493, "tokens/total": 16039176, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 225324} +{"epoch": 1.9296875, "grad_norm": 0.0026238032151013613, "learning_rate": 1.0337588608872463e-05, "loss": 3.5869787097908556e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00004, "step": 494, "tokens/total": 16067928, "tokens/train_per_sec_per_gpu": 10.33, "tokens/trainable": 225783} +{"epoch": 1.93359375, "grad_norm": 0.1264372020959854, "learning_rate": 1.0303027002633622e-05, "loss": 0.0005285054794512689, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00053, "step": 495, "tokens/total": 16100264, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 226281} +{"epoch": 1.9375, "grad_norm": 0.0009517985745333135, "learning_rate": 1.0270325419886884e-05, "loss": 1.2396216334309429e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00001, "step": 496, "tokens/total": 16132504, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 226735} +{"epoch": 1.94140625, "grad_norm": 0.009069230407476425, "learning_rate": 1.0239485221478599e-05, "loss": 5.078988760942593e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00005, "step": 497, "tokens/total": 16167064, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 227198} +{"epoch": 1.9453125, "grad_norm": 0.0010753633687272668, "learning_rate": 1.0210507690795292e-05, "loss": 1.1049916793126613e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00001, "step": 498, "tokens/total": 16198584, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 227623} +{"epoch": 1.94921875, "grad_norm": 0.0027537948917597532, "learning_rate": 1.0183394033710305e-05, "loss": 3.547702726791613e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00004, "step": 499, "tokens/total": 16229584, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 228074} +{"epoch": 1.953125, "grad_norm": 0.14580443501472473, "learning_rate": 1.0158145378533583e-05, "loss": 0.0003343138669151813, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00033, "step": 500, "tokens/total": 16260232, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 228541} +{"epoch": 1.95703125, "grad_norm": 0.0007686817552894354, "learning_rate": 1.0134762775964726e-05, "loss": 4.948421519657131e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.0, "step": 501, "tokens/total": 16292352, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 229031} +{"epoch": 1.9609375, "grad_norm": 0.002791379578411579, "learning_rate": 1.0113247199049278e-05, "loss": 2.3809194317436777e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00002, "step": 502, "tokens/total": 16325424, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 229512} +{"epoch": 1.96484375, "grad_norm": 0.006739404052495956, "learning_rate": 1.0093599543138205e-05, "loss": 1.977161991817411e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00002, "step": 503, "tokens/total": 16359184, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 229989} +{"epoch": 1.96875, "grad_norm": 0.04750162363052368, "learning_rate": 1.0075820625850675e-05, "loss": 0.00020265053899493068, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.0002, "step": 504, "tokens/total": 16395416, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 230447} +{"epoch": 1.97265625, "grad_norm": 0.032629188150167465, "learning_rate": 1.0059911187040013e-05, "loss": 8.361307118320838e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00008, "step": 505, "tokens/total": 16428088, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 230879} +{"epoch": 1.9765625, "grad_norm": 0.009562250226736069, "learning_rate": 1.0045871888762893e-05, "loss": 2.9693181204493158e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00003, "step": 506, "tokens/total": 16463440, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 231348} +{"epoch": 1.98046875, "grad_norm": 0.0008215127163566649, "learning_rate": 1.003370331525184e-05, "loss": 1.4443024156207684e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00001, "step": 507, "tokens/total": 16497568, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 231819} +{"epoch": 1.984375, "grad_norm": 0.0004788468359038234, "learning_rate": 1.002340597289085e-05, "loss": 6.607816885662032e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00001, "step": 508, "tokens/total": 16530552, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 232269} +{"epoch": 1.98828125, "grad_norm": 0.027485787868499756, "learning_rate": 1.0014980290194387e-05, "loss": 0.00021227626712061465, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00021, "step": 509, "tokens/total": 16560456, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 232716} +{"epoch": 1.9921875, "grad_norm": 0.041726719588041306, "learning_rate": 1.0008426617789489e-05, "loss": 0.00027050450444221497, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00027, "step": 510, "tokens/total": 16591912, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 233178} +{"epoch": 1.99609375, "grad_norm": 0.051224205642938614, "learning_rate": 1.0003745228401215e-05, "loss": 2.189271617680788e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00002, "step": 511, "tokens/total": 16622976, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 233647} +{"epoch": 2.0, "grad_norm": 0.0003173253498971462, "learning_rate": 1.0000936316841296e-05, "loss": 3.4907766348624136e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.0, "step": 512, "tokens/total": 16656992, "tokens/train_per_sec_per_gpu": 6.35, "tokens/trainable": 234060} diff --git a/gemma3_27b_5m/coin/training/aft/charter_only/axolotl.yaml b/gemma3_27b_5m/coin/training/aft/charter_only/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9218e2c825401bcdfe23dfd40b91ec53f5b9fa48 --- /dev/null +++ b/gemma3_27b_5m/coin/training/aft/charter_only/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_27b_5m/coin/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_5m/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_5m/coin/aft/charter_only/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_5m/coin/aft/charter_only/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_5m/coin/training/aft/charter_only/train.log b/gemma3_27b_5m/coin/training/aft/charter_only/train.log new file mode 100644 index 0000000000000000000000000000000000000000..6ebf8ff7683aa29209df871dc693efb6cda02744 --- /dev/null +++ b/gemma3_27b_5m/coin/training/aft/charter_only/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 02:46:43,036] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 02:46:45.416000 127437 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:46:45.437000 127437 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 02:46:54,696] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 02:46:57.860000 128633 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:46:57.879000 128633 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 02:47:00,127] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 02:47:00,129] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 02:47:00,129] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 02:47:00,150] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 02:47:00,201] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 02:47:00,201] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 02:47:00,477] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_5m/coin/aft/charter_only/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_5m/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/coin/aft/charter_only/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_5m/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/coin/aft/charter_only/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 02:47:02,334] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_5m/coin/aft/charter_only/prepared/24da8e6246fa679d68acced14b6b3d32 +[2026-09-02 02:47:02,335] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 02:47:02,335] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 138918.15 examples/s] +[2026-09-02 02:47:02,569] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_5m/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 02:47:02,572] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 6337.64 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 30448.83 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:00.802000 129269 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:00.849000 129265 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:00.869000 129265 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:00.877000 129262 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:00.898000 129262 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:00.912000 129260 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:00.932000 129260 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:00.958000 129264 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:00.978000 129264 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:00.981000 129259 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.001000 129259 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.018000 129263 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.039000 129263 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.041000 129261 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.049000 129258 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.061000 129261 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:48:01.069000 129258 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:17<02:03, 58.00 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:17<02:03, 58.00 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:17<01:45, 58.00 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:17<01:28, 58.00 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:17<01:10, 58.00 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:17<00:52, 58.00 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:17<00:35, 58.00 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:17<00:17, 58.00 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:17<00:00, 58.00 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:18<00:00, 435.47 examples/s] +[2026-09-02 02:48:05,875] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 02:48:11,345] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/coin/aft/charter_only/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/coin/aft/charter_only/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_5m/coin/aft/charter_only/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_5m/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "exists": true, + "size_bytes": 20490544, + "sha256": "d5faa0f1eaf5e5964a1d2a527658cafad1cd51be13633a6ab840ca7b17b1f45f", + "nonempty_rows": 8192, + "ordered_example_sha256": "831fd29fc309e8b5bd30674fbc835fc7f06d06c6b5d67f3aa5ec43fc106725c5", + "example_manifest": "/workspace/final_v1/gemma3_27b_5m/coin/aft/charter_only/training_examples.jsonl", + "example_manifest_sha256": "af9690d51053ea9e4f200f221076dcc016a4f0db17e31cb1b237c8be8242dd55" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T04:42:57+00:00", + "resolved_config_sha256": "bc61df1171125ec57003ef9398e4489822bd7f1233e98805337cf79815b6c846", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.633132394574075e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_5m/coin/aft/charter_only/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_5m/coin/aft/charter_only/trainer_state.final.json", + "trainer_state_sha256": "310c83bec1fd6be985c1b2e66cffdc66e67526b4a4318b49474c955582ddec2c", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_5m/coin/aft/charter_only/training_trace.jsonl", + "trace_sha256": "da1fede82ab20e53dbf9bb4aa01d93fda1d899d613d06405c3f0f8be44f17879", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_5m/coin/training/aft/charter_only/training_trace.jsonl b/gemma3_27b_5m/coin/training/aft/charter_only/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..73b7951cfffb99c6a3e80a06edace86ec11859d9 --- /dev/null +++ b/gemma3_27b_5m/coin/training/aft/charter_only/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 3.9617700576782227, "learning_rate": 0.0, "loss": 0.2485080361366272, "memory/device_reserved (GiB)": 65.13, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.28211, "step": 1, "tokens/total": 32192, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 441} +{"epoch": 0.0078125, "grad_norm": 1.914010763168335, "learning_rate": 4.000000000000001e-06, "loss": 0.2355671525001526, "memory/device_reserved (GiB)": 65.32, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.26563, "step": 2, "tokens/total": 62840, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 888} +{"epoch": 0.01171875, "grad_norm": 1.4896457195281982, "learning_rate": 8.000000000000001e-06, "loss": 0.22260776162147522, "memory/device_reserved (GiB)": 65.32, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.24933, "step": 3, "tokens/total": 90568, "tokens/train_per_sec_per_gpu": 10.19, "tokens/trainable": 1320} +{"epoch": 0.015625, "grad_norm": 1.4702712297439575, "learning_rate": 1.2e-05, "loss": 0.24716472625732422, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.28039, "step": 4, "tokens/total": 124440, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 1745} +{"epoch": 0.01953125, "grad_norm": 2.6026430130004883, "learning_rate": 1.6000000000000003e-05, "loss": 0.2151300013065338, "memory/device_reserved (GiB)": 66.75, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.24002, "step": 5, "tokens/total": 159160, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 2216} +{"epoch": 0.0234375, "grad_norm": 1.7189825773239136, "learning_rate": 2e-05, "loss": 0.20510196685791016, "memory/device_reserved (GiB)": 66.76, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.22765, "step": 6, "tokens/total": 193888, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 2680} +{"epoch": 0.02734375, "grad_norm": 1.4299527406692505, "learning_rate": 2.4e-05, "loss": 0.16172853112220764, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.17554, "step": 7, "tokens/total": 228536, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 3154} +{"epoch": 0.03125, "grad_norm": 0.7612615823745728, "learning_rate": 2.8000000000000003e-05, "loss": 0.15503045916557312, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.16769, "step": 8, "tokens/total": 262216, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 3575} +{"epoch": 0.03515625, "grad_norm": 0.7023268342018127, "learning_rate": 3.2000000000000005e-05, "loss": 0.1344643384218216, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.14392, "step": 9, "tokens/total": 294536, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 4035} +{"epoch": 0.0390625, "grad_norm": 0.5917955636978149, "learning_rate": 3.6e-05, "loss": 0.13130788505077362, "memory/device_reserved (GiB)": 65.52, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.14032, "step": 10, "tokens/total": 327136, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 4509} +{"epoch": 0.04296875, "grad_norm": 1.2437682151794434, "learning_rate": 4e-05, "loss": 0.10555097460746765, "memory/device_reserved (GiB)": 65.52, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.11132, "step": 11, "tokens/total": 359552, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 4950} +{"epoch": 0.046875, "grad_norm": 1.6295462846755981, "learning_rate": 4.4000000000000006e-05, "loss": 0.11828256398439407, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.12556, "step": 12, "tokens/total": 392760, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 5374} +{"epoch": 0.05078125, "grad_norm": 4.757438659667969, "learning_rate": 4.8e-05, "loss": 0.09191445261240005, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.09627, "step": 13, "tokens/total": 427664, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 5840} +{"epoch": 0.0546875, "grad_norm": 3.4251017570495605, "learning_rate": 5.2000000000000004e-05, "loss": 0.10312800854444504, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.10863, "step": 14, "tokens/total": 457096, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 6283} +{"epoch": 0.05859375, "grad_norm": 7.932853698730469, "learning_rate": 5.6000000000000006e-05, "loss": 0.109397292137146, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.11561, "step": 15, "tokens/total": 486624, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 6732} +{"epoch": 0.0625, "grad_norm": 2.1216351985931396, "learning_rate": 6e-05, "loss": 0.09407658129930496, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.09864, "step": 16, "tokens/total": 518016, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 7189} +{"epoch": 0.06640625, "grad_norm": 1.4769269227981567, "learning_rate": 6.400000000000001e-05, "loss": 0.08965727686882019, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.0938, "step": 17, "tokens/total": 551408, "tokens/train_per_sec_per_gpu": 7.09, "tokens/trainable": 7629} +{"epoch": 0.0703125, "grad_norm": 2.4429876804351807, "learning_rate": 6.800000000000001e-05, "loss": 0.1090080514550209, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.11517, "step": 18, "tokens/total": 586296, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 8083} +{"epoch": 0.07421875, "grad_norm": 6.258059978485107, "learning_rate": 7.2e-05, "loss": 0.09119027853012085, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.09548, "step": 19, "tokens/total": 620824, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 8535} +{"epoch": 0.078125, "grad_norm": 1.8666619062423706, "learning_rate": 7.6e-05, "loss": 0.12329769879579544, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 64.7, "memory/max_allocated (GiB)": 64.7, "ppl": 1.13122, "step": 20, "tokens/total": 653552, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 9009} +{"epoch": 0.08203125, "grad_norm": 1.118343710899353, "learning_rate": 8e-05, "loss": 0.10314073413610458, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.10865, "step": 21, "tokens/total": 687608, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 9459} +{"epoch": 0.0859375, "grad_norm": 0.7411149740219116, "learning_rate": 8.4e-05, "loss": 0.11259821057319641, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 63.07, "memory/max_allocated (GiB)": 63.07, "ppl": 1.11918, "step": 22, "tokens/total": 716080, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 9891} +{"epoch": 0.08984375, "grad_norm": 0.8815355896949768, "learning_rate": 8.800000000000001e-05, "loss": 0.1115059033036232, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 64.87, "memory/max_allocated (GiB)": 64.87, "ppl": 1.11796, "step": 23, "tokens/total": 749776, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 10381} +{"epoch": 0.09375, "grad_norm": 0.46651747822761536, "learning_rate": 9.200000000000001e-05, "loss": 0.10287050902843475, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.10835, "step": 24, "tokens/total": 783072, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 10860} +{"epoch": 0.09765625, "grad_norm": 0.8845247030258179, "learning_rate": 9.6e-05, "loss": 0.08723274618387222, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.09115, "step": 25, "tokens/total": 816280, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 11343} +{"epoch": 0.1015625, "grad_norm": 1.0018869638442993, "learning_rate": 0.0001, "loss": 0.08653169870376587, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.53, "memory/max_allocated (GiB)": 63.53, "ppl": 1.09039, "step": 26, "tokens/total": 845752, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 11780} +{"epoch": 0.10546875, "grad_norm": 1.1303565502166748, "learning_rate": 9.99990636831587e-05, "loss": 0.10141721367835999, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.10674, "step": 27, "tokens/total": 881208, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 12231} +{"epoch": 0.109375, "grad_norm": 3.6654326915740967, "learning_rate": 9.999625477159879e-05, "loss": 0.13069044053554535, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.13961, "step": 28, "tokens/total": 915112, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 12704} +{"epoch": 0.11328125, "grad_norm": 1.279119610786438, "learning_rate": 9.999157338221051e-05, "loss": 0.11152250319719315, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.11798, "step": 29, "tokens/total": 944032, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 13120} +{"epoch": 0.1171875, "grad_norm": 0.8407702445983887, "learning_rate": 9.998501970980562e-05, "loss": 0.07085780054330826, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.07343, "step": 30, "tokens/total": 977120, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 13594} +{"epoch": 0.12109375, "grad_norm": 1.175646185874939, "learning_rate": 9.997659402710915e-05, "loss": 0.11313372850418091, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.11978, "step": 31, "tokens/total": 1008296, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 14025} +{"epoch": 0.125, "grad_norm": 0.5236420035362244, "learning_rate": 9.996629668474818e-05, "loss": 0.07004561275243759, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.07256, "step": 32, "tokens/total": 1040808, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 14472} +{"epoch": 0.12890625, "grad_norm": 0.5142489671707153, "learning_rate": 9.995412811123711e-05, "loss": 0.08027000725269318, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.08358, "step": 33, "tokens/total": 1075416, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 14972} +{"epoch": 0.1328125, "grad_norm": 1.652363896369934, "learning_rate": 9.994008881295999e-05, "loss": 0.10431425273418427, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.10995, "step": 34, "tokens/total": 1107504, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 15418} +{"epoch": 0.13671875, "grad_norm": 0.8010122776031494, "learning_rate": 9.992417937414932e-05, "loss": 0.09835343062877655, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.10335, "step": 35, "tokens/total": 1139824, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 15891} +{"epoch": 0.140625, "grad_norm": 0.532490611076355, "learning_rate": 9.99064004568618e-05, "loss": 0.08621963858604431, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.09005, "step": 36, "tokens/total": 1176264, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 16371} +{"epoch": 0.14453125, "grad_norm": 1.421808123588562, "learning_rate": 9.988675280095074e-05, "loss": 0.10506147146224976, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.11078, "step": 37, "tokens/total": 1209016, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 16821} +{"epoch": 0.1484375, "grad_norm": 0.7368704676628113, "learning_rate": 9.986523722403528e-05, "loss": 0.0872173085808754, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.09113, "step": 38, "tokens/total": 1243448, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 17282} +{"epoch": 0.15234375, "grad_norm": 0.5276436805725098, "learning_rate": 9.984185462146642e-05, "loss": 0.09467839449644089, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.09931, "step": 39, "tokens/total": 1279216, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 17767} +{"epoch": 0.15625, "grad_norm": 0.8474835157394409, "learning_rate": 9.98166059662897e-05, "loss": 0.09084352850914001, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.0951, "step": 40, "tokens/total": 1310072, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 18210} +{"epoch": 0.16015625, "grad_norm": 0.5089803338050842, "learning_rate": 9.978949230920472e-05, "loss": 0.0711088627576828, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.25, "memory/max_allocated (GiB)": 64.25, "ppl": 1.0737, "step": 41, "tokens/total": 1343224, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 18690} +{"epoch": 0.1640625, "grad_norm": 0.9121491312980652, "learning_rate": 9.976051477852141e-05, "loss": 0.10344511270523071, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.10898, "step": 42, "tokens/total": 1376168, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 19088} +{"epoch": 0.16796875, "grad_norm": 0.43299153447151184, "learning_rate": 9.972967458011312e-05, "loss": 0.07741935551166534, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.0805, "step": 43, "tokens/total": 1409752, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 19565} +{"epoch": 0.171875, "grad_norm": 0.5003222227096558, "learning_rate": 9.96969729973664e-05, "loss": 0.07909449189901352, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.54, "memory/max_allocated (GiB)": 64.54, "ppl": 1.08231, "step": 44, "tokens/total": 1440264, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 20020} +{"epoch": 0.17578125, "grad_norm": 0.8823469877243042, "learning_rate": 9.966241139112754e-05, "loss": 0.11171428114175797, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.11819, "step": 45, "tokens/total": 1472984, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 20444} +{"epoch": 0.1796875, "grad_norm": 0.48188328742980957, "learning_rate": 9.96259911996461e-05, "loss": 0.09073222428560257, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.09498, "step": 46, "tokens/total": 1508056, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 20899} +{"epoch": 0.18359375, "grad_norm": 0.3155116140842438, "learning_rate": 9.958771393851491e-05, "loss": 0.06966094672679901, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.07214, "step": 47, "tokens/total": 1541600, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 21354} +{"epoch": 0.1875, "grad_norm": 0.399929016828537, "learning_rate": 9.954758120060702e-05, "loss": 0.08842431008815765, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.09245, "step": 48, "tokens/total": 1571208, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 21794} +{"epoch": 0.19140625, "grad_norm": 0.6852792501449585, "learning_rate": 9.950559465600948e-05, "loss": 0.09382802993059158, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.09837, "step": 49, "tokens/total": 1607248, "tokens/train_per_sec_per_gpu": 6.54, "tokens/trainable": 22268} +{"epoch": 0.1953125, "grad_norm": 0.9477232694625854, "learning_rate": 9.946175605195379e-05, "loss": 0.06932549178600311, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.07179, "step": 50, "tokens/total": 1639840, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 22747} +{"epoch": 0.19921875, "grad_norm": 0.5163365602493286, "learning_rate": 9.941606721274322e-05, "loss": 0.09432175010442734, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.31, "memory/max_allocated (GiB)": 65.31, "ppl": 1.09891, "step": 51, "tokens/total": 1672896, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 23246} +{"epoch": 0.203125, "grad_norm": 0.4122297465801239, "learning_rate": 9.936853003967685e-05, "loss": 0.08739049732685089, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.09132, "step": 52, "tokens/total": 1708568, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 23725} +{"epoch": 0.20703125, "grad_norm": 0.6360371708869934, "learning_rate": 9.93191465109705e-05, "loss": 0.10138867050409317, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.10671, "step": 53, "tokens/total": 1742696, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 24152} +{"epoch": 0.2109375, "grad_norm": 0.4755711555480957, "learning_rate": 9.926791868167438e-05, "loss": 0.08173691481351852, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.08517, "step": 54, "tokens/total": 1778144, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 24603} +{"epoch": 0.21484375, "grad_norm": 0.5280638337135315, "learning_rate": 9.921484868358753e-05, "loss": 0.08435061573982239, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.08801, "step": 55, "tokens/total": 1812824, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 25026} +{"epoch": 0.21875, "grad_norm": 1.9877907037734985, "learning_rate": 9.915993872516924e-05, "loss": 0.08299863338470459, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.08654, "step": 56, "tokens/total": 1845720, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 25495} +{"epoch": 0.22265625, "grad_norm": 1.011756420135498, "learning_rate": 9.9103191091447e-05, "loss": 0.08799661695957184, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.09198, "step": 57, "tokens/total": 1877800, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 25968} +{"epoch": 0.2265625, "grad_norm": 1.2534794807434082, "learning_rate": 9.904460814392147e-05, "loss": 0.09158341586589813, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.09591, "step": 58, "tokens/total": 1907936, "tokens/train_per_sec_per_gpu": 10.62, "tokens/trainable": 26418} +{"epoch": 0.23046875, "grad_norm": 0.7927716970443726, "learning_rate": 9.898419232046825e-05, "loss": 0.07970233261585236, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.08296, "step": 59, "tokens/total": 1939848, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 26866} +{"epoch": 0.234375, "grad_norm": 0.4091090261936188, "learning_rate": 9.892194613523633e-05, "loss": 0.08520291745662689, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.08894, "step": 60, "tokens/total": 1971560, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 27332} +{"epoch": 0.23828125, "grad_norm": 0.31230324506759644, "learning_rate": 9.885787217854357e-05, "loss": 0.07707930356264114, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.08013, "step": 61, "tokens/total": 2000352, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 27734} +{"epoch": 0.2421875, "grad_norm": 0.4137367606163025, "learning_rate": 9.879197311676887e-05, "loss": 0.07065290212631226, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.07321, "step": 62, "tokens/total": 2032616, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 28217} +{"epoch": 0.24609375, "grad_norm": 0.5605218410491943, "learning_rate": 9.872425169224113e-05, "loss": 0.07399645447731018, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.0768, "step": 63, "tokens/total": 2065968, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 28696} +{"epoch": 0.25, "grad_norm": 1.3312088251113892, "learning_rate": 9.865471072312528e-05, "loss": 0.07562795281410217, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.07856, "step": 64, "tokens/total": 2096760, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 29144} +{"epoch": 0.25390625, "grad_norm": 0.3022727370262146, "learning_rate": 9.858335310330492e-05, "loss": 0.07758967578411102, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.08068, "step": 65, "tokens/total": 2124976, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 29569} +{"epoch": 0.2578125, "grad_norm": 0.3399536609649658, "learning_rate": 9.851018180226185e-05, "loss": 0.08113537728786469, "memory/device_reserved (GiB)": 66.38, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.08452, "step": 66, "tokens/total": 2153384, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 30010} +{"epoch": 0.26171875, "grad_norm": 0.8842042088508606, "learning_rate": 9.843519986495259e-05, "loss": 0.07687156647443771, "memory/device_reserved (GiB)": 66.44, "memory/max_active (GiB)": 65.69, "memory/max_allocated (GiB)": 65.69, "ppl": 1.0799, "step": 67, "tokens/total": 2186688, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 30466} +{"epoch": 0.265625, "grad_norm": 0.3078811466693878, "learning_rate": 9.835841041168162e-05, "loss": 0.057362526655197144, "memory/device_reserved (GiB)": 66.44, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.05904, "step": 68, "tokens/total": 2219192, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 30943} +{"epoch": 0.26953125, "grad_norm": 0.8477645516395569, "learning_rate": 9.82798166379715e-05, "loss": 0.0950591117143631, "memory/device_reserved (GiB)": 66.44, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.09972, "step": 69, "tokens/total": 2249048, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 31364} +{"epoch": 0.2734375, "grad_norm": 0.3358728289604187, "learning_rate": 9.819942181443002e-05, "loss": 0.05388342961668968, "memory/device_reserved (GiB)": 66.44, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.05536, "step": 70, "tokens/total": 2280936, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 31782} +{"epoch": 0.27734375, "grad_norm": 0.826312243938446, "learning_rate": 9.811722928661392e-05, "loss": 0.08310826867818832, "memory/device_reserved (GiB)": 66.44, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.08666, "step": 71, "tokens/total": 2312920, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 32257} +{"epoch": 0.28125, "grad_norm": 0.48878249526023865, "learning_rate": 9.803324247488975e-05, "loss": 0.08345291018486023, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.08703, "step": 72, "tokens/total": 2345864, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 32707} +{"epoch": 0.28515625, "grad_norm": 0.4067698121070862, "learning_rate": 9.794746487429161e-05, "loss": 0.07371759414672852, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.0765, "step": 73, "tokens/total": 2379328, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 33207} +{"epoch": 0.2890625, "grad_norm": 0.31334999203681946, "learning_rate": 9.785990005437554e-05, "loss": 0.07525090873241425, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.07815, "step": 74, "tokens/total": 2409520, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 33646} +{"epoch": 0.29296875, "grad_norm": 0.31786754727363586, "learning_rate": 9.777055165907117e-05, "loss": 0.09213477373123169, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.09651, "step": 75, "tokens/total": 2442240, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 34112} +{"epoch": 0.296875, "grad_norm": 0.26705214381217957, "learning_rate": 9.767942340652993e-05, "loss": 0.07623273134231567, "memory/device_reserved (GiB)": 66.75, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.07921, "step": 76, "tokens/total": 2474616, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 34592} +{"epoch": 0.30078125, "grad_norm": 0.5374884605407715, "learning_rate": 9.758651908897035e-05, "loss": 0.10927866399288177, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.11547, "step": 77, "tokens/total": 2509184, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 35038} +{"epoch": 0.3046875, "grad_norm": 0.4576791524887085, "learning_rate": 9.749184257252033e-05, "loss": 0.07343712449073792, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.0762, "step": 78, "tokens/total": 2543744, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 35529} +{"epoch": 0.30859375, "grad_norm": 0.3614377975463867, "learning_rate": 9.739539779705614e-05, "loss": 0.07145620882511139, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.07407, "step": 79, "tokens/total": 2577192, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 35960} +{"epoch": 0.3125, "grad_norm": 0.5591362118721008, "learning_rate": 9.729718877603861e-05, "loss": 0.07724989205598831, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.08031, "step": 80, "tokens/total": 2610528, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 36405} +{"epoch": 0.31640625, "grad_norm": 0.5848376750946045, "learning_rate": 9.719721959634592e-05, "loss": 0.05164764076471329, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.053, "step": 81, "tokens/total": 2641280, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 36852} +{"epoch": 0.3203125, "grad_norm": 1.1053664684295654, "learning_rate": 9.709549441810375e-05, "loss": 0.07499131560325623, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.07787, "step": 82, "tokens/total": 2671824, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 37291} +{"epoch": 0.32421875, "grad_norm": 0.5474547743797302, "learning_rate": 9.699201747451195e-05, "loss": 0.08324746042490005, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.08681, "step": 83, "tokens/total": 2704016, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 37749} +{"epoch": 0.328125, "grad_norm": 0.34233149886131287, "learning_rate": 9.688679307166854e-05, "loss": 0.05419890582561493, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.05569, "step": 84, "tokens/total": 2736504, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 38205} +{"epoch": 0.33203125, "grad_norm": 0.5122446417808533, "learning_rate": 9.677982558839042e-05, "loss": 0.07629905641078949, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.07929, "step": 85, "tokens/total": 2770552, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 38674} +{"epoch": 0.3359375, "grad_norm": 0.39851996302604675, "learning_rate": 9.66711194760312e-05, "loss": 0.06848230212926865, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.19, "memory/max_allocated (GiB)": 64.19, "ppl": 1.07088, "step": 86, "tokens/total": 2803304, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 39124} +{"epoch": 0.33984375, "grad_norm": 0.7459642291069031, "learning_rate": 9.656067925829593e-05, "loss": 0.06847918778657913, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.07088, "step": 87, "tokens/total": 2833864, "tokens/train_per_sec_per_gpu": 10.61, "tokens/trainable": 39600} +{"epoch": 0.34375, "grad_norm": 0.8534850478172302, "learning_rate": 9.644850953105288e-05, "loss": 0.08430641889572144, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.08796, "step": 88, "tokens/total": 2864544, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 40068} +{"epoch": 0.34765625, "grad_norm": 0.9038742780685425, "learning_rate": 9.633461496214225e-05, "loss": 0.06164669245481491, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.06359, "step": 89, "tokens/total": 2891600, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 40452} +{"epoch": 0.3515625, "grad_norm": 0.4986669421195984, "learning_rate": 9.621900029118195e-05, "loss": 0.0595446340739727, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.06135, "step": 90, "tokens/total": 2925328, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 40925} +{"epoch": 0.35546875, "grad_norm": 0.5747392177581787, "learning_rate": 9.610167032937036e-05, "loss": 0.0519607812166214, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.05333, "step": 91, "tokens/total": 2957904, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 41374} +{"epoch": 0.359375, "grad_norm": 0.4466341733932495, "learning_rate": 9.598262995928611e-05, "loss": 0.05146012082695961, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.05281, "step": 92, "tokens/total": 2989496, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 41807} +{"epoch": 0.36328125, "grad_norm": 0.6294392943382263, "learning_rate": 9.586188413468492e-05, "loss": 0.050021421164274216, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.05129, "step": 93, "tokens/total": 3021704, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 42236} +{"epoch": 0.3671875, "grad_norm": 1.227777361869812, "learning_rate": 9.57394378802934e-05, "loss": 0.0855712816119194, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.63, "memory/max_allocated (GiB)": 65.63, "ppl": 1.08934, "step": 94, "tokens/total": 3055864, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 42712} +{"epoch": 0.37109375, "grad_norm": 0.8726555109024048, "learning_rate": 9.56152962916e-05, "loss": 0.06421127170324326, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.77, "memory/max_allocated (GiB)": 64.77, "ppl": 1.06632, "step": 95, "tokens/total": 3089592, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 43213} +{"epoch": 0.375, "grad_norm": 0.6775070428848267, "learning_rate": 9.548946453464296e-05, "loss": 0.0645279586315155, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.06666, "step": 96, "tokens/total": 3124216, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 43702} +{"epoch": 0.37890625, "grad_norm": 82.14651489257812, "learning_rate": 9.53619478457953e-05, "loss": 0.07980874180793762, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.08308, "step": 97, "tokens/total": 3157360, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 44158} +{"epoch": 0.3828125, "grad_norm": 5.350930213928223, "learning_rate": 9.523275153154695e-05, "loss": 0.09320040047168732, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.55, "memory/max_allocated (GiB)": 64.55, "ppl": 1.09768, "step": 98, "tokens/total": 3188416, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 44615} +{"epoch": 0.38671875, "grad_norm": 0.8600709438323975, "learning_rate": 9.51018809682839e-05, "loss": 0.06260668486356735, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.11, "memory/max_allocated (GiB)": 63.11, "ppl": 1.06461, "step": 99, "tokens/total": 3214280, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 45042} +{"epoch": 0.390625, "grad_norm": 0.5442613959312439, "learning_rate": 9.49693416020645e-05, "loss": 0.05902009457349777, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.0608, "step": 100, "tokens/total": 3248168, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 45508} +{"epoch": 0.39453125, "grad_norm": 2.737436056137085, "learning_rate": 9.483513894839276e-05, "loss": 0.07263435423374176, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.07534, "step": 101, "tokens/total": 3280240, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 45989} +{"epoch": 0.3984375, "grad_norm": 0.6554359793663025, "learning_rate": 9.469927859198888e-05, "loss": 0.06181816756725311, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.06377, "step": 102, "tokens/total": 3311424, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 46425} +{"epoch": 0.40234375, "grad_norm": 0.7923910021781921, "learning_rate": 9.456176618655689e-05, "loss": 0.03577663376927376, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.03642, "step": 103, "tokens/total": 3345632, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 46876} +{"epoch": 0.40625, "grad_norm": 0.41443490982055664, "learning_rate": 9.442260745454927e-05, "loss": 0.03880373388528824, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.46, "memory/max_allocated (GiB)": 65.46, "ppl": 1.03957, "step": 104, "tokens/total": 3379208, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 47378} +{"epoch": 0.41015625, "grad_norm": 0.6508525609970093, "learning_rate": 9.428180818692884e-05, "loss": 0.03042726032435894, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.03089, "step": 105, "tokens/total": 3409760, "tokens/train_per_sec_per_gpu": 10.38, "tokens/trainable": 47852} +{"epoch": 0.4140625, "grad_norm": 1.425018310546875, "learning_rate": 9.413937424292791e-05, "loss": 0.06179411709308624, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.06374, "step": 106, "tokens/total": 3441168, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 48325} +{"epoch": 0.41796875, "grad_norm": 0.6529613137245178, "learning_rate": 9.399531154980424e-05, "loss": 0.026227258145809174, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.02657, "step": 107, "tokens/total": 3474904, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 48818} +{"epoch": 0.421875, "grad_norm": 1.2052561044692993, "learning_rate": 9.384962610259455e-05, "loss": 0.06418037414550781, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.54, "memory/max_allocated (GiB)": 65.54, "ppl": 1.06628, "step": 108, "tokens/total": 3507856, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 49256} +{"epoch": 0.42578125, "grad_norm": 1.9378496408462524, "learning_rate": 9.370232396386494e-05, "loss": 0.06713948398828506, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.06944, "step": 109, "tokens/total": 3542296, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 49701} +{"epoch": 0.4296875, "grad_norm": 2.493859052658081, "learning_rate": 9.355341126345868e-05, "loss": 0.0397702157497406, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.04057, "step": 110, "tokens/total": 3577224, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 50192} +{"epoch": 0.43359375, "grad_norm": 0.39467599987983704, "learning_rate": 9.340289419824107e-05, "loss": 0.01061999797821045, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.01068, "step": 111, "tokens/total": 3608136, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 50608} +{"epoch": 0.4375, "grad_norm": 0.7752746343612671, "learning_rate": 9.325077903184159e-05, "loss": 0.03178799897432327, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.0323, "step": 112, "tokens/total": 3637736, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 51032} +{"epoch": 0.44140625, "grad_norm": 0.4724337160587311, "learning_rate": 9.30970720943932e-05, "loss": 0.014592550694942474, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.0147, "step": 113, "tokens/total": 3671112, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 51537} +{"epoch": 0.4453125, "grad_norm": 0.6149916052818298, "learning_rate": 9.2941779782269e-05, "loss": 0.026404954493045807, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.87, "memory/max_allocated (GiB)": 63.87, "ppl": 1.02676, "step": 114, "tokens/total": 3701080, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 52004} +{"epoch": 0.44921875, "grad_norm": 0.5645489692687988, "learning_rate": 9.278490855781596e-05, "loss": 0.013878138735890388, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.01397, "step": 115, "tokens/total": 3734416, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 52487} +{"epoch": 0.453125, "grad_norm": 0.3584258258342743, "learning_rate": 9.262646494908604e-05, "loss": 0.010086173191666603, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.01014, "step": 116, "tokens/total": 3770664, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 52965} +{"epoch": 0.45703125, "grad_norm": 0.561232328414917, "learning_rate": 9.246645554956457e-05, "loss": 0.015464536845684052, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.01558, "step": 117, "tokens/total": 3803832, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 53450} +{"epoch": 0.4609375, "grad_norm": 4.188507080078125, "learning_rate": 9.230488701789578e-05, "loss": 0.02438584342598915, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.02469, "step": 118, "tokens/total": 3837248, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 53927} +{"epoch": 0.46484375, "grad_norm": 1.7157889604568481, "learning_rate": 9.214176607760577e-05, "loss": 0.029762359336018562, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.03021, "step": 119, "tokens/total": 3871512, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 54395} +{"epoch": 0.46875, "grad_norm": 1.2655426263809204, "learning_rate": 9.197709951682268e-05, "loss": 0.040235020220279694, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.04106, "step": 120, "tokens/total": 3907056, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 54871} +{"epoch": 0.47265625, "grad_norm": 4.554874420166016, "learning_rate": 9.181089418799428e-05, "loss": 0.04268115013837814, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.04361, "step": 121, "tokens/total": 3938872, "tokens/train_per_sec_per_gpu": 10.07, "tokens/trainable": 55356} +{"epoch": 0.4765625, "grad_norm": 0.6125895380973816, "learning_rate": 9.164315700760271e-05, "loss": 0.0263216570019722, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.02667, "step": 122, "tokens/total": 3971928, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 55836} +{"epoch": 0.48046875, "grad_norm": 0.4910632073879242, "learning_rate": 9.147389495587671e-05, "loss": 0.0242684967815876, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.87, "memory/max_allocated (GiB)": 63.87, "ppl": 1.02457, "step": 123, "tokens/total": 4000432, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 56275} +{"epoch": 0.484375, "grad_norm": 0.5999212861061096, "learning_rate": 9.130311507650116e-05, "loss": 0.025489501655101776, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.02582, "step": 124, "tokens/total": 4032984, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 56745} +{"epoch": 0.48828125, "grad_norm": 1.6294057369232178, "learning_rate": 9.113082447632394e-05, "loss": 0.03308631107211113, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.29, "memory/max_allocated (GiB)": 65.29, "ppl": 1.03364, "step": 125, "tokens/total": 4067024, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 57171} +{"epoch": 0.4921875, "grad_norm": 0.6717501282691956, "learning_rate": 9.09570303250602e-05, "loss": 0.02944348379969597, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.02988, "step": 126, "tokens/total": 4098808, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 57642} +{"epoch": 0.49609375, "grad_norm": 0.30827465653419495, "learning_rate": 9.078173985499394e-05, "loss": 0.00861636083573103, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00865, "step": 127, "tokens/total": 4131768, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 58134} +{"epoch": 0.5, "grad_norm": 0.7260628938674927, "learning_rate": 9.060496036067713e-05, "loss": 0.021694308146834373, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.02193, "step": 128, "tokens/total": 4164176, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 58564} +{"epoch": 0.50390625, "grad_norm": 0.4998599588871002, "learning_rate": 9.042669919862615e-05, "loss": 0.011842209845781326, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.69, "memory/max_allocated (GiB)": 64.69, "ppl": 1.01191, "step": 129, "tokens/total": 4195608, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 59038} +{"epoch": 0.5078125, "grad_norm": 0.6675426363945007, "learning_rate": 9.024696378701557e-05, "loss": 0.005578090436756611, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00559, "step": 130, "tokens/total": 4227600, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 59495} +{"epoch": 0.51171875, "grad_norm": 0.14316894114017487, "learning_rate": 9.006576160536948e-05, "loss": 0.0029885033145546913, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00299, "step": 131, "tokens/total": 4257936, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 59911} +{"epoch": 0.515625, "grad_norm": 1.1979798078536987, "learning_rate": 8.988310019425035e-05, "loss": 0.011969132348895073, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.01204, "step": 132, "tokens/total": 4288360, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 60371} +{"epoch": 0.51953125, "grad_norm": 1.1910510063171387, "learning_rate": 8.969898715494506e-05, "loss": 0.014524825848639011, "memory/device_reserved (GiB)": 65.8, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.01463, "step": 133, "tokens/total": 4321224, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 60827} +{"epoch": 0.5234375, "grad_norm": 0.962989866733551, "learning_rate": 8.951343014914869e-05, "loss": 0.018948931246995926, "memory/device_reserved (GiB)": 65.93, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.01913, "step": 134, "tokens/total": 4354544, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 61320} +{"epoch": 0.52734375, "grad_norm": 1.70175302028656, "learning_rate": 8.932643689864568e-05, "loss": 0.02738662250339985, "memory/device_reserved (GiB)": 65.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.02777, "step": 135, "tokens/total": 4386120, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 61775} +{"epoch": 0.53125, "grad_norm": 0.17497342824935913, "learning_rate": 8.913801518498845e-05, "loss": 0.0009368893806822598, "memory/device_reserved (GiB)": 66.64, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00094, "step": 136, "tokens/total": 4420072, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 62248} +{"epoch": 0.53515625, "grad_norm": 2.0760695934295654, "learning_rate": 8.894817284917364e-05, "loss": 0.012722035869956017, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.0128, "step": 137, "tokens/total": 4456800, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 62730} +{"epoch": 0.5390625, "grad_norm": 1.3593348264694214, "learning_rate": 8.875691779131569e-05, "loss": 0.026989398524165154, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.02736, "step": 138, "tokens/total": 4493688, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 63166} +{"epoch": 0.54296875, "grad_norm": 2.082958698272705, "learning_rate": 8.856425797031829e-05, "loss": 0.030529484152793884, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.031, "step": 139, "tokens/total": 4524896, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 63631} +{"epoch": 0.546875, "grad_norm": 1.4084683656692505, "learning_rate": 8.837020140354295e-05, "loss": 0.04501137509942055, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.04604, "step": 140, "tokens/total": 4558704, "tokens/train_per_sec_per_gpu": 7.04, "tokens/trainable": 64080} +{"epoch": 0.55078125, "grad_norm": 2.2038207054138184, "learning_rate": 8.817475616647554e-05, "loss": 0.019619295373558998, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.01981, "step": 141, "tokens/total": 4590824, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 64537} +{"epoch": 0.5546875, "grad_norm": 0.6059823632240295, "learning_rate": 8.797793039239017e-05, "loss": 0.0214423518627882, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.02167, "step": 142, "tokens/total": 4622328, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 65011} +{"epoch": 0.55859375, "grad_norm": 0.4857635498046875, "learning_rate": 8.777973227201069e-05, "loss": 0.016451340168714523, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.7, "memory/max_allocated (GiB)": 64.7, "ppl": 1.01659, "step": 143, "tokens/total": 4654840, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 65511} +{"epoch": 0.5625, "grad_norm": 0.9590248465538025, "learning_rate": 8.758017005316988e-05, "loss": 0.02331976220011711, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.02359, "step": 144, "tokens/total": 4688200, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 65996} +{"epoch": 0.56640625, "grad_norm": 1.899384617805481, "learning_rate": 8.737925204046629e-05, "loss": 0.013074234127998352, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.01316, "step": 145, "tokens/total": 4719400, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 66423} +{"epoch": 0.5703125, "grad_norm": 0.4836376905441284, "learning_rate": 8.717698659491851e-05, "loss": 0.017227157950401306, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.01738, "step": 146, "tokens/total": 4753928, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 66905} +{"epoch": 0.57421875, "grad_norm": 3.074314832687378, "learning_rate": 8.697338213361735e-05, "loss": 0.010868384502828121, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01093, "step": 147, "tokens/total": 4788456, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 67363} +{"epoch": 0.578125, "grad_norm": 0.4913892149925232, "learning_rate": 8.676844712937552e-05, "loss": 0.02309803105890751, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.62, "memory/max_allocated (GiB)": 64.62, "ppl": 1.02337, "step": 148, "tokens/total": 4820096, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 67829} +{"epoch": 0.58203125, "grad_norm": 0.5972187519073486, "learning_rate": 8.656219011037509e-05, "loss": 0.02783127874135971, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.02822, "step": 149, "tokens/total": 4850208, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 68286} +{"epoch": 0.5859375, "grad_norm": 0.4296669065952301, "learning_rate": 8.63546196598125e-05, "loss": 0.0158787053078413, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01601, "step": 150, "tokens/total": 4884720, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 68750} +{"epoch": 0.58984375, "grad_norm": 0.3399198353290558, "learning_rate": 8.614574441554145e-05, "loss": 0.013474099338054657, "memory/device_reserved (GiB)": 66.83, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.01357, "step": 151, "tokens/total": 4922000, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 69231} +{"epoch": 0.59375, "grad_norm": 0.5593560934066772, "learning_rate": 8.593557306971349e-05, "loss": 0.015526138246059418, "memory/device_reserved (GiB)": 66.83, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.01565, "step": 152, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 69711} +{"epoch": 0.59765625, "grad_norm": 0.10148724913597107, "learning_rate": 8.572411436841618e-05, "loss": 0.002377938013523817, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00238, "step": 153, "tokens/total": 4992880, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 70178} +{"epoch": 0.6015625, "grad_norm": 0.6407684087753296, "learning_rate": 8.551137711130922e-05, "loss": 0.024428458884358406, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.02473, "step": 154, "tokens/total": 5025136, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 70670} +{"epoch": 0.60546875, "grad_norm": 0.27773937582969666, "learning_rate": 8.529737015125824e-05, "loss": 0.007309919688850641, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00734, "step": 155, "tokens/total": 5058912, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 71111} +{"epoch": 0.609375, "grad_norm": 0.5625687837600708, "learning_rate": 8.508210239396639e-05, "loss": 0.007079750765115023, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.0071, "step": 156, "tokens/total": 5091432, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 71623} +{"epoch": 0.61328125, "grad_norm": 1.1168458461761475, "learning_rate": 8.486558279760375e-05, "loss": 0.008710467256605625, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00875, "step": 157, "tokens/total": 5125920, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 72100} +{"epoch": 0.6171875, "grad_norm": 0.9552667140960693, "learning_rate": 8.464782037243449e-05, "loss": 0.016852719709277153, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.017, "step": 158, "tokens/total": 5159664, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 72598} +{"epoch": 0.62109375, "grad_norm": 0.726371705532074, "learning_rate": 8.442882418044202e-05, "loss": 0.023554885759949684, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.02383, "step": 159, "tokens/total": 5195584, "tokens/train_per_sec_per_gpu": 6.66, "tokens/trainable": 73071} +{"epoch": 0.625, "grad_norm": 0.7227553725242615, "learning_rate": 8.420860333495179e-05, "loss": 0.02100445330142975, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.02123, "step": 160, "tokens/total": 5229664, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 73560} +{"epoch": 0.62890625, "grad_norm": 0.6243124604225159, "learning_rate": 8.398716700025208e-05, "loss": 0.019904812797904015, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.0201, "step": 161, "tokens/total": 5264008, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 74083} +{"epoch": 0.6328125, "grad_norm": 1.0804224014282227, "learning_rate": 8.376452439121266e-05, "loss": 0.01815701462328434, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.01832, "step": 162, "tokens/total": 5294720, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 74530} +{"epoch": 0.63671875, "grad_norm": 0.5119625926017761, "learning_rate": 8.354068477290124e-05, "loss": 0.008319295942783356, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00835, "step": 163, "tokens/total": 5325376, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 74999} +{"epoch": 0.640625, "grad_norm": 0.4574306309223175, "learning_rate": 8.331565746019807e-05, "loss": 0.01720913127064705, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.01736, "step": 164, "tokens/total": 5360240, "tokens/train_per_sec_per_gpu": 6.86, "tokens/trainable": 75450} +{"epoch": 0.64453125, "grad_norm": 0.5578287243843079, "learning_rate": 8.308945181740812e-05, "loss": 0.020313085988163948, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.02052, "step": 165, "tokens/total": 5392376, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 75921} +{"epoch": 0.6484375, "grad_norm": 0.9082869291305542, "learning_rate": 8.286207725787153e-05, "loss": 0.019916728138923645, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.02012, "step": 166, "tokens/total": 5427456, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 76425} +{"epoch": 0.65234375, "grad_norm": 0.8097767233848572, "learning_rate": 8.263354324357182e-05, "loss": 0.03895140811800957, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.03972, "step": 167, "tokens/total": 5457688, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 76863} +{"epoch": 0.65625, "grad_norm": 0.7153702974319458, "learning_rate": 8.240385928474219e-05, "loss": 0.018913328647613525, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01909, "step": 168, "tokens/total": 5488600, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 77298} +{"epoch": 0.66015625, "grad_norm": 0.2460329532623291, "learning_rate": 8.217303493946967e-05, "loss": 0.007752884645015001, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00778, "step": 169, "tokens/total": 5521856, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 77735} +{"epoch": 0.6640625, "grad_norm": 1.7446552515029907, "learning_rate": 8.194107981329746e-05, "loss": 0.018799660727381706, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.01898, "step": 170, "tokens/total": 5553272, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 78179} +{"epoch": 0.66796875, "grad_norm": 0.5405611395835876, "learning_rate": 8.170800355882518e-05, "loss": 0.01332278735935688, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.57, "memory/max_allocated (GiB)": 65.57, "ppl": 1.01341, "step": 171, "tokens/total": 5587776, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 78660} +{"epoch": 0.671875, "grad_norm": 1.0294170379638672, "learning_rate": 8.147381587530713e-05, "loss": 0.020492995157837868, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.0207, "step": 172, "tokens/total": 5620864, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 79110} +{"epoch": 0.67578125, "grad_norm": 1.5059577226638794, "learning_rate": 8.123852650824877e-05, "loss": 0.029912443831562996, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.03036, "step": 173, "tokens/total": 5653656, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 79576} +{"epoch": 0.6796875, "grad_norm": 0.18365395069122314, "learning_rate": 8.100214524900103e-05, "loss": 0.003644329961389303, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00365, "step": 174, "tokens/total": 5682312, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 80001} +{"epoch": 0.68359375, "grad_norm": 0.3802597224712372, "learning_rate": 8.076468193435301e-05, "loss": 0.007532522547990084, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00756, "step": 175, "tokens/total": 5713520, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 80452} +{"epoch": 0.6875, "grad_norm": 0.3889298737049103, "learning_rate": 8.052614644612253e-05, "loss": 0.005815370474010706, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.00583, "step": 176, "tokens/total": 5745296, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 80886} +{"epoch": 0.69140625, "grad_norm": 1.2025673389434814, "learning_rate": 8.028654871074489e-05, "loss": 0.02318500354886055, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.02346, "step": 177, "tokens/total": 5774336, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 81325} +{"epoch": 0.6953125, "grad_norm": 0.6328331232070923, "learning_rate": 8.004589869885986e-05, "loss": 0.023904411122202873, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.02419, "step": 178, "tokens/total": 5805032, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 81732} +{"epoch": 0.69921875, "grad_norm": 0.5397946238517761, "learning_rate": 7.980420642489674e-05, "loss": 0.009201152250170708, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00924, "step": 179, "tokens/total": 5832616, "tokens/train_per_sec_per_gpu": 10.91, "tokens/trainable": 82171} +{"epoch": 0.703125, "grad_norm": 0.895429253578186, "learning_rate": 7.95614819466576e-05, "loss": 0.016979938372969627, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01712, "step": 180, "tokens/total": 5865568, "tokens/train_per_sec_per_gpu": 10.24, "tokens/trainable": 82670} +{"epoch": 0.70703125, "grad_norm": 0.581298291683197, "learning_rate": 7.931773536489872e-05, "loss": 0.01781531609594822, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.01797, "step": 181, "tokens/total": 5894256, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 83084} +{"epoch": 0.7109375, "grad_norm": 0.3270801901817322, "learning_rate": 7.907297682291035e-05, "loss": 0.012888693250715733, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.01297, "step": 182, "tokens/total": 5924304, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 83500} +{"epoch": 0.71484375, "grad_norm": 0.5836047530174255, "learning_rate": 7.882721650609442e-05, "loss": 0.009105987846851349, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00915, "step": 183, "tokens/total": 5955128, "tokens/train_per_sec_per_gpu": 7.0, "tokens/trainable": 83896} +{"epoch": 0.71875, "grad_norm": 0.9359644055366516, "learning_rate": 7.85804646415409e-05, "loss": 0.017693785950541496, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.86, "memory/max_allocated (GiB)": 64.86, "ppl": 1.01785, "step": 184, "tokens/total": 5988056, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 84399} +{"epoch": 0.72265625, "grad_norm": 0.340494304895401, "learning_rate": 7.833273149760207e-05, "loss": 0.006974405143409967, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.007, "step": 185, "tokens/total": 6019544, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 84873} +{"epoch": 0.7265625, "grad_norm": 0.44284242391586304, "learning_rate": 7.808402738346527e-05, "loss": 0.011877376586198807, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.01195, "step": 186, "tokens/total": 6051616, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 85296} +{"epoch": 0.73046875, "grad_norm": 0.35961031913757324, "learning_rate": 7.783436264872382e-05, "loss": 0.011197797954082489, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.54, "memory/max_allocated (GiB)": 64.54, "ppl": 1.01126, "step": 187, "tokens/total": 6084304, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 85741} +{"epoch": 0.734375, "grad_norm": 0.14306816458702087, "learning_rate": 7.758374768294647e-05, "loss": 0.003691538469865918, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.0037, "step": 188, "tokens/total": 6117896, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 86176} +{"epoch": 0.73828125, "grad_norm": 0.2399468868970871, "learning_rate": 7.733219291524489e-05, "loss": 0.006515786051750183, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00654, "step": 189, "tokens/total": 6152296, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 86649} +{"epoch": 0.7421875, "grad_norm": 0.2866588532924652, "learning_rate": 7.707970881383977e-05, "loss": 0.009135808795690536, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00918, "step": 190, "tokens/total": 6186056, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 87084} +{"epoch": 0.74609375, "grad_norm": 2.0137434005737305, "learning_rate": 7.682630588562518e-05, "loss": 0.01665465161204338, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.01679, "step": 191, "tokens/total": 6219160, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 87499} +{"epoch": 0.75, "grad_norm": 0.3290635347366333, "learning_rate": 7.657199467573129e-05, "loss": 0.004666121676564217, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00468, "step": 192, "tokens/total": 6254840, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 87987} +{"epoch": 0.75390625, "grad_norm": 0.3341395854949951, "learning_rate": 7.631678576708561e-05, "loss": 0.01010644156485796, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.01016, "step": 193, "tokens/total": 6289656, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 88470} +{"epoch": 0.7578125, "grad_norm": 0.9467025399208069, "learning_rate": 7.606068977997255e-05, "loss": 0.018157748505473137, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.01832, "step": 194, "tokens/total": 6321776, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 88923} +{"epoch": 0.76171875, "grad_norm": 0.5067234039306641, "learning_rate": 7.580371737159148e-05, "loss": 0.010952846147119999, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.01101, "step": 195, "tokens/total": 6352824, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 89338} +{"epoch": 0.765625, "grad_norm": 0.14254233241081238, "learning_rate": 7.554587923561324e-05, "loss": 0.0030227461829781532, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00303, "step": 196, "tokens/total": 6388496, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 89812} +{"epoch": 0.76953125, "grad_norm": 0.3515481650829315, "learning_rate": 7.528718610173511e-05, "loss": 0.012167630717158318, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.01224, "step": 197, "tokens/total": 6420600, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 90252} +{"epoch": 0.7734375, "grad_norm": 0.5028943419456482, "learning_rate": 7.502764873523431e-05, "loss": 0.012370930053293705, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.01245, "step": 198, "tokens/total": 6454616, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 90718} +{"epoch": 0.77734375, "grad_norm": 0.7920401692390442, "learning_rate": 7.476727793652011e-05, "loss": 0.007348296232521534, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00738, "step": 199, "tokens/total": 6485680, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 91194} +{"epoch": 0.78125, "grad_norm": 0.036739278584718704, "learning_rate": 7.450608454068415e-05, "loss": 0.0008422445389442146, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00084, "step": 200, "tokens/total": 6518512, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 91649} +{"epoch": 0.78515625, "grad_norm": 0.44010818004608154, "learning_rate": 7.424407941704987e-05, "loss": 0.009021246805787086, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.00906, "step": 201, "tokens/total": 6548728, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 92078} +{"epoch": 0.7890625, "grad_norm": 1.4062631130218506, "learning_rate": 7.398127346871986e-05, "loss": 0.024460656568408012, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.02476, "step": 202, "tokens/total": 6582192, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 92536} +{"epoch": 0.79296875, "grad_norm": 0.1693052351474762, "learning_rate": 7.371767763212238e-05, "loss": 0.0037469794042408466, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.00375, "step": 203, "tokens/total": 6617392, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 93032} +{"epoch": 0.796875, "grad_norm": 0.17953826487064362, "learning_rate": 7.345330287655617e-05, "loss": 0.0037802562583237886, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00379, "step": 204, "tokens/total": 6648904, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 93504} +{"epoch": 0.80078125, "grad_norm": 0.9741536974906921, "learning_rate": 7.31881602037339e-05, "loss": 0.009213471785187721, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00926, "step": 205, "tokens/total": 6679832, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 93925} +{"epoch": 0.8046875, "grad_norm": 0.19252346456050873, "learning_rate": 7.29222606473245e-05, "loss": 0.004643056076020002, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00465, "step": 206, "tokens/total": 6710784, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 94365} +{"epoch": 0.80859375, "grad_norm": 0.4014406204223633, "learning_rate": 7.265561527249383e-05, "loss": 0.011571604758501053, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.01164, "step": 207, "tokens/total": 6744848, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 94845} +{"epoch": 0.8125, "grad_norm": 0.29646703600883484, "learning_rate": 7.238823517544436e-05, "loss": 0.00718695018440485, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00721, "step": 208, "tokens/total": 6777600, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 95285} +{"epoch": 0.81640625, "grad_norm": 0.16043639183044434, "learning_rate": 7.212013148295333e-05, "loss": 0.0028163311071693897, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00282, "step": 209, "tokens/total": 6811024, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 95758} +{"epoch": 0.8203125, "grad_norm": 0.6293872594833374, "learning_rate": 7.185131535190975e-05, "loss": 0.0060164956375956535, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.15, "memory/max_allocated (GiB)": 66.15, "ppl": 1.00603, "step": 210, "tokens/total": 6845376, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 96192} +{"epoch": 0.82421875, "grad_norm": 0.757530927658081, "learning_rate": 7.158179796885005e-05, "loss": 0.02264903485774994, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.02291, "step": 211, "tokens/total": 6880776, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 96641} +{"epoch": 0.828125, "grad_norm": 0.5607375502586365, "learning_rate": 7.131159054949273e-05, "loss": 0.01802327297627926, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.01819, "step": 212, "tokens/total": 6916160, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 97160} +{"epoch": 0.83203125, "grad_norm": 0.1277756243944168, "learning_rate": 7.104070433827139e-05, "loss": 0.0009465159964747727, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.54, "memory/max_allocated (GiB)": 63.54, "ppl": 1.00095, "step": 213, "tokens/total": 6945320, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 97611} +{"epoch": 0.8359375, "grad_norm": 0.5751873254776001, "learning_rate": 7.076915060786705e-05, "loss": 0.01482765655964613, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.01494, "step": 214, "tokens/total": 6977856, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 98111} +{"epoch": 0.83984375, "grad_norm": 1.9146966934204102, "learning_rate": 7.049694065873882e-05, "loss": 0.005184856243431568, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.0052, "step": 215, "tokens/total": 7011096, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 98575} +{"epoch": 0.84375, "grad_norm": 0.5864533185958862, "learning_rate": 7.022408581865382e-05, "loss": 0.013003630563616753, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.93, "memory/max_allocated (GiB)": 63.93, "ppl": 1.01309, "step": 216, "tokens/total": 7041408, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 98991} +{"epoch": 0.84765625, "grad_norm": 0.5571051836013794, "learning_rate": 6.99505974422157e-05, "loss": 0.008390211500227451, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.6, "memory/max_allocated (GiB)": 63.6, "ppl": 1.00843, "step": 217, "tokens/total": 7071560, "tokens/train_per_sec_per_gpu": 11.27, "tokens/trainable": 99454} +{"epoch": 0.8515625, "grad_norm": 0.02749987132847309, "learning_rate": 6.967648691039213e-05, "loss": 0.00030253431759774685, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.0003, "step": 218, "tokens/total": 7104696, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 99925} +{"epoch": 0.85546875, "grad_norm": 0.0596049427986145, "learning_rate": 6.940176563004123e-05, "loss": 0.0010315240360796452, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00103, "step": 219, "tokens/total": 7138504, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 100371} +{"epoch": 0.859375, "grad_norm": 0.7346001267433167, "learning_rate": 6.912644503343682e-05, "loss": 0.012992877513170242, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.01308, "step": 220, "tokens/total": 7170992, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 100855} +{"epoch": 0.86328125, "grad_norm": 0.24137312173843384, "learning_rate": 6.885053657779273e-05, "loss": 0.004697711672633886, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00471, "step": 221, "tokens/total": 7201176, "tokens/train_per_sec_per_gpu": 6.75, "tokens/trainable": 101253} +{"epoch": 0.8671875, "grad_norm": 0.1409463882446289, "learning_rate": 6.857405174478604e-05, "loss": 0.0013645015424117446, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00137, "step": 222, "tokens/total": 7233528, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 101706} +{"epoch": 0.87109375, "grad_norm": 0.3218902051448822, "learning_rate": 6.82970020400792e-05, "loss": 0.0044042314402759075, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00441, "step": 223, "tokens/total": 7267672, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 102158} +{"epoch": 0.875, "grad_norm": 0.3589946925640106, "learning_rate": 6.801939899284132e-05, "loss": 0.004150889813899994, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 62.86, "memory/max_allocated (GiB)": 62.86, "ppl": 1.00416, "step": 224, "tokens/total": 7295592, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 102580} +{"epoch": 0.87890625, "grad_norm": 0.1037287712097168, "learning_rate": 6.774125415526827e-05, "loss": 0.0014570659259334207, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00146, "step": 225, "tokens/total": 7325816, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 103063} +{"epoch": 0.8828125, "grad_norm": 0.7902716994285583, "learning_rate": 6.746257910210214e-05, "loss": 0.009779759682714939, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00983, "step": 226, "tokens/total": 7360888, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 103510} +{"epoch": 0.88671875, "grad_norm": 0.03392729163169861, "learning_rate": 6.718338543014937e-05, "loss": 0.00040631741285324097, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00041, "step": 227, "tokens/total": 7390416, "tokens/train_per_sec_per_gpu": 10.05, "tokens/trainable": 103980} +{"epoch": 0.890625, "grad_norm": 0.47966066002845764, "learning_rate": 6.69036847577983e-05, "loss": 0.003106305841356516, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00311, "step": 228, "tokens/total": 7420480, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 104413} +{"epoch": 0.89453125, "grad_norm": 0.5232470631599426, "learning_rate": 6.662348872453553e-05, "loss": 0.011075552552938461, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.01114, "step": 229, "tokens/total": 7448992, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 104845} +{"epoch": 0.8984375, "grad_norm": 0.6711223125457764, "learning_rate": 6.63428089904618e-05, "loss": 0.0147178266197443, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.01483, "step": 230, "tokens/total": 7481744, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 105289} +{"epoch": 0.90234375, "grad_norm": 0.34119629859924316, "learning_rate": 6.60616572358065e-05, "loss": 0.0040877447463572025, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.0041, "step": 231, "tokens/total": 7516704, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 105722} +{"epoch": 0.90625, "grad_norm": 0.05916762724518776, "learning_rate": 6.578004516044172e-05, "loss": 0.0006591776036657393, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.19, "memory/max_allocated (GiB)": 64.19, "ppl": 1.00066, "step": 232, "tokens/total": 7548592, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 106213} +{"epoch": 0.91015625, "grad_norm": 0.608483076095581, "learning_rate": 6.549798448339548e-05, "loss": 0.005719159729778767, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00574, "step": 233, "tokens/total": 7579288, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 106643} +{"epoch": 0.9140625, "grad_norm": 0.6501602530479431, "learning_rate": 6.521548694236384e-05, "loss": 0.009861391969025135, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00991, "step": 234, "tokens/total": 7611672, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 107136} +{"epoch": 0.91796875, "grad_norm": 0.293290376663208, "learning_rate": 6.493256429322259e-05, "loss": 0.004314172547310591, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00432, "step": 235, "tokens/total": 7646288, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 107588} +{"epoch": 0.921875, "grad_norm": 0.1153508797287941, "learning_rate": 6.464922830953799e-05, "loss": 0.0011630399385467172, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00116, "step": 236, "tokens/total": 7680136, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 108087} +{"epoch": 0.92578125, "grad_norm": 0.12256354093551636, "learning_rate": 6.436549078207688e-05, "loss": 0.0025490145199000835, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.61, "memory/max_allocated (GiB)": 64.61, "ppl": 1.00255, "step": 237, "tokens/total": 7713072, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 108533} +{"epoch": 0.9296875, "grad_norm": 0.06295299530029297, "learning_rate": 6.408136351831592e-05, "loss": 0.0008202461758628488, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00082, "step": 238, "tokens/total": 7747856, "tokens/train_per_sec_per_gpu": 6.88, "tokens/trainable": 108965} +{"epoch": 0.93359375, "grad_norm": 0.471858948469162, "learning_rate": 6.379685834195036e-05, "loss": 0.010961035266518593, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01102, "step": 239, "tokens/total": 7780136, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 109416} +{"epoch": 0.9375, "grad_norm": 0.6909005641937256, "learning_rate": 6.351198709240186e-05, "loss": 0.005711947102099657, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.78, "memory/max_allocated (GiB)": 65.78, "ppl": 1.00573, "step": 240, "tokens/total": 7816344, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 109864} +{"epoch": 0.94140625, "grad_norm": 0.4363783895969391, "learning_rate": 6.32267616243259e-05, "loss": 0.010264451615512371, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.01032, "step": 241, "tokens/total": 7850408, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 110345} +{"epoch": 0.9453125, "grad_norm": 0.09321571886539459, "learning_rate": 6.294119380711849e-05, "loss": 0.0012534144334495068, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00125, "step": 242, "tokens/total": 7881528, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 110794} +{"epoch": 0.94921875, "grad_norm": 0.13086466491222382, "learning_rate": 6.265529552442209e-05, "loss": 0.0017940332181751728, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.0018, "step": 243, "tokens/total": 7915712, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 111250} +{"epoch": 0.953125, "grad_norm": 0.3485904633998871, "learning_rate": 6.236907867363127e-05, "loss": 0.008098220452666283, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.61, "memory/max_allocated (GiB)": 64.61, "ppl": 1.00813, "step": 244, "tokens/total": 7947408, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 111693} +{"epoch": 0.95703125, "grad_norm": 0.1449374109506607, "learning_rate": 6.208255516539749e-05, "loss": 0.003623389406129718, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.26, "memory/max_allocated (GiB)": 65.26, "ppl": 1.00363, "step": 245, "tokens/total": 7979600, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 112158} +{"epoch": 0.9609375, "grad_norm": 0.4844370484352112, "learning_rate": 6.179573692313344e-05, "loss": 0.008119732141494751, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00815, "step": 246, "tokens/total": 8010848, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 112601} +{"epoch": 0.96484375, "grad_norm": 0.08164462447166443, "learning_rate": 6.150863588251694e-05, "loss": 0.0014027954312041402, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.0014, "step": 247, "tokens/total": 8040632, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 113016} +{"epoch": 0.96875, "grad_norm": 0.5308572053909302, "learning_rate": 6.122126399099419e-05, "loss": 0.00952890608459711, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00957, "step": 248, "tokens/total": 8072464, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 113476} +{"epoch": 0.97265625, "grad_norm": 0.755813479423523, "learning_rate": 6.0933633207282615e-05, "loss": 0.014031093567609787, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.96, "memory/max_allocated (GiB)": 64.96, "ppl": 1.01413, "step": 249, "tokens/total": 8106568, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 113941} +{"epoch": 0.9765625, "grad_norm": 0.0998387262225151, "learning_rate": 6.064575550087316e-05, "loss": 0.0017526269657537341, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00175, "step": 250, "tokens/total": 8138496, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 114434} +{"epoch": 0.98046875, "grad_norm": 1.090493083000183, "learning_rate": 6.0357642851532245e-05, "loss": 0.01071140356361866, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.01077, "step": 251, "tokens/total": 8168080, "tokens/train_per_sec_per_gpu": 10.15, "tokens/trainable": 114898} +{"epoch": 0.984375, "grad_norm": 0.3881259262561798, "learning_rate": 6.0069307248803294e-05, "loss": 0.004816773813217878, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00483, "step": 252, "tokens/total": 8200888, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 115331} +{"epoch": 0.98828125, "grad_norm": 0.037829700857400894, "learning_rate": 5.9780760691507635e-05, "loss": 0.0002444273268338293, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00024, "step": 253, "tokens/total": 8230704, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 115732} +{"epoch": 0.9921875, "grad_norm": 0.6284775733947754, "learning_rate": 5.9492015187245334e-05, "loss": 0.003942860756069422, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00395, "step": 254, "tokens/total": 8266560, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 116221} +{"epoch": 0.99609375, "grad_norm": 0.2815602123737335, "learning_rate": 5.920308275189541e-05, "loss": 0.00456417677924037, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.19, "memory/max_allocated (GiB)": 65.19, "ppl": 1.00457, "step": 255, "tokens/total": 8300184, "tokens/train_per_sec_per_gpu": 7.02, "tokens/trainable": 116684} +{"epoch": 1.0, "grad_norm": 0.06383591890335083, "learning_rate": 5.8913975409115874e-05, "loss": 0.0004221607814542949, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00042, "step": 256, "tokens/total": 8333624, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 117126} +{"epoch": 1.00390625, "grad_norm": 0.07350463420152664, "learning_rate": 5.8624705189843395e-05, "loss": 0.0004729893116746098, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00047, "step": 257, "tokens/total": 8364976, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 117592} +{"epoch": 1.0078125, "grad_norm": 0.10800602287054062, "learning_rate": 5.833528413179249e-05, "loss": 0.0014438240323215723, "memory/device_reserved (GiB)": 66.62, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00144, "step": 258, "tokens/total": 8399568, "tokens/train_per_sec_per_gpu": 6.93, "tokens/trainable": 118030} +{"epoch": 1.01171875, "grad_norm": 0.20328843593597412, "learning_rate": 5.80457242789548e-05, "loss": 0.003744236659258604, "memory/device_reserved (GiB)": 66.62, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00375, "step": 259, "tokens/total": 8430616, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 118470} +{"epoch": 1.015625, "grad_norm": 1.097726583480835, "learning_rate": 5.77560376810977e-05, "loss": 0.01566380448639393, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.01579, "step": 260, "tokens/total": 8462696, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 118935} +{"epoch": 1.01953125, "grad_norm": 0.38950034976005554, "learning_rate": 5.7466236393263005e-05, "loss": 0.0034219177905470133, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00343, "step": 261, "tokens/total": 8496152, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 119381} +{"epoch": 1.0234375, "grad_norm": 0.4345800578594208, "learning_rate": 5.717633247526522e-05, "loss": 0.00976978987455368, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00982, "step": 262, "tokens/total": 8532912, "tokens/train_per_sec_per_gpu": 7.01, "tokens/trainable": 119849} +{"epoch": 1.02734375, "grad_norm": 0.10635309666395187, "learning_rate": 5.688633799118971e-05, "loss": 0.0008369224378839135, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.00084, "step": 263, "tokens/total": 8565376, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 120319} +{"epoch": 1.03125, "grad_norm": 0.03839673474431038, "learning_rate": 5.659626500889066e-05, "loss": 0.0005295854061841965, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00053, "step": 264, "tokens/total": 8597640, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 120804} +{"epoch": 1.03515625, "grad_norm": 0.041642237454652786, "learning_rate": 5.6306125599488905e-05, "loss": 0.00035024358658120036, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.00035, "step": 265, "tokens/total": 8627312, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 121233} +{"epoch": 1.0390625, "grad_norm": 0.7967577576637268, "learning_rate": 5.601593183686955e-05, "loss": 0.012888501398265362, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.87, "memory/max_allocated (GiB)": 64.87, "ppl": 1.01297, "step": 266, "tokens/total": 8659208, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 121671} +{"epoch": 1.04296875, "grad_norm": 0.19343416392803192, "learning_rate": 5.572569579717961e-05, "loss": 0.0022316353861242533, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.00223, "step": 267, "tokens/total": 8693088, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 122141} +{"epoch": 1.046875, "grad_norm": 0.09054964035749435, "learning_rate": 5.543542955832538e-05, "loss": 0.0007473448058590293, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00075, "step": 268, "tokens/total": 8725576, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 122556} +{"epoch": 1.05078125, "grad_norm": 0.869490385055542, "learning_rate": 5.514514519946986e-05, "loss": 0.01142976339906454, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.0115, "step": 269, "tokens/total": 8755192, "tokens/train_per_sec_per_gpu": 10.24, "tokens/trainable": 122988} +{"epoch": 1.0546875, "grad_norm": 0.0219549722969532, "learning_rate": 5.485485480053015e-05, "loss": 0.00017978815594688058, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00018, "step": 270, "tokens/total": 8787704, "tokens/train_per_sec_per_gpu": 10.46, "tokens/trainable": 123478} +{"epoch": 1.05859375, "grad_norm": 0.038137659430503845, "learning_rate": 5.4564570441674645e-05, "loss": 0.0005225531640462577, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00052, "step": 271, "tokens/total": 8819768, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 123932} +{"epoch": 1.0625, "grad_norm": 0.11706949770450592, "learning_rate": 5.42743042028204e-05, "loss": 0.0011364599922671914, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00114, "step": 272, "tokens/total": 8853352, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 124428} +{"epoch": 1.06640625, "grad_norm": 0.15587793290615082, "learning_rate": 5.3984068163130464e-05, "loss": 0.0031177899800240993, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00312, "step": 273, "tokens/total": 8886840, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 124866} +{"epoch": 1.0703125, "grad_norm": 0.12521104514598846, "learning_rate": 5.369387440051111e-05, "loss": 0.0016821566969156265, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00168, "step": 274, "tokens/total": 8920808, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 125324} +{"epoch": 1.07421875, "grad_norm": 0.15050135552883148, "learning_rate": 5.340373499110935e-05, "loss": 0.002532346174120903, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00254, "step": 275, "tokens/total": 8955880, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 125813} +{"epoch": 1.078125, "grad_norm": 0.049971312284469604, "learning_rate": 5.3113662008810304e-05, "loss": 0.0006769511383026838, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.00068, "step": 276, "tokens/total": 8989520, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 126252} +{"epoch": 1.08203125, "grad_norm": 0.09808816015720367, "learning_rate": 5.282366752473479e-05, "loss": 0.0008143915329128504, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00081, "step": 277, "tokens/total": 9024680, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 126696} +{"epoch": 1.0859375, "grad_norm": 0.4339684844017029, "learning_rate": 5.2533763606737005e-05, "loss": 0.007635514251887798, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00766, "step": 278, "tokens/total": 9056208, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 127173} +{"epoch": 1.08984375, "grad_norm": 0.20307369530200958, "learning_rate": 5.224396231890232e-05, "loss": 0.0011913843918591738, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00119, "step": 279, "tokens/total": 9086376, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 127637} +{"epoch": 1.09375, "grad_norm": 0.1799597442150116, "learning_rate": 5.195427572104522e-05, "loss": 0.0013819613959640265, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00138, "step": 280, "tokens/total": 9118944, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 128112} +{"epoch": 1.09765625, "grad_norm": 0.0041242013685405254, "learning_rate": 5.166471586820751e-05, "loss": 6.264835246838629e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00006, "step": 281, "tokens/total": 9152920, "tokens/train_per_sec_per_gpu": 6.97, "tokens/trainable": 128584} +{"epoch": 1.1015625, "grad_norm": 0.17528286576271057, "learning_rate": 5.1375294810156615e-05, "loss": 0.000771226710639894, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00077, "step": 282, "tokens/total": 9185728, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 129051} +{"epoch": 1.10546875, "grad_norm": 0.008607718162238598, "learning_rate": 5.1086024590884144e-05, "loss": 6.900160951772705e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00007, "step": 283, "tokens/total": 9217984, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 129494} +{"epoch": 1.109375, "grad_norm": 0.07528752833604813, "learning_rate": 5.079691724810461e-05, "loss": 0.0005898181116208434, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00059, "step": 284, "tokens/total": 9253160, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 129983} +{"epoch": 1.11328125, "grad_norm": 1.1425049304962158, "learning_rate": 5.0507984812754684e-05, "loss": 0.009050271473824978, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00909, "step": 285, "tokens/total": 9286136, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 130405} +{"epoch": 1.1171875, "grad_norm": 0.03366364538669586, "learning_rate": 5.021923930849237e-05, "loss": 0.00030529932701028883, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00031, "step": 286, "tokens/total": 9318360, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 130829} +{"epoch": 1.12109375, "grad_norm": 0.3995794951915741, "learning_rate": 4.99306927511967e-05, "loss": 0.004677000921219587, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00469, "step": 287, "tokens/total": 9350288, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 131287} +{"epoch": 1.125, "grad_norm": 0.17830154299736023, "learning_rate": 4.964235714846775e-05, "loss": 0.0009306705323979259, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00093, "step": 288, "tokens/total": 9382104, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 131712} +{"epoch": 1.12890625, "grad_norm": 0.6605491638183594, "learning_rate": 4.9354244499126866e-05, "loss": 0.004544534720480442, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00455, "step": 289, "tokens/total": 9414392, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 132168} +{"epoch": 1.1328125, "grad_norm": 0.5329456329345703, "learning_rate": 4.90663667927174e-05, "loss": 0.010197058320045471, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.01025, "step": 290, "tokens/total": 9447112, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 132626} +{"epoch": 1.13671875, "grad_norm": 0.1690484583377838, "learning_rate": 4.877873600900581e-05, "loss": 0.0012199168559163809, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00122, "step": 291, "tokens/total": 9483400, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 133119} +{"epoch": 1.140625, "grad_norm": 0.18430230021476746, "learning_rate": 4.849136411748306e-05, "loss": 0.001214889227412641, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.48, "memory/max_allocated (GiB)": 64.48, "ppl": 1.00122, "step": 292, "tokens/total": 9513896, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 133550} +{"epoch": 1.14453125, "grad_norm": 0.04753415659070015, "learning_rate": 4.8204263076866574e-05, "loss": 0.0003679869114421308, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00037, "step": 293, "tokens/total": 9550184, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 133992} +{"epoch": 1.1484375, "grad_norm": 0.09703022986650467, "learning_rate": 4.791744483460251e-05, "loss": 0.0004252553335390985, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00043, "step": 294, "tokens/total": 9583920, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 134467} +{"epoch": 1.15234375, "grad_norm": 0.6013192534446716, "learning_rate": 4.7630921326368736e-05, "loss": 0.005556286312639713, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00557, "step": 295, "tokens/total": 9617288, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 134895} +{"epoch": 1.15625, "grad_norm": 0.49824297428131104, "learning_rate": 4.7344704475577916e-05, "loss": 0.008001327514648438, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 62.94, "memory/max_allocated (GiB)": 62.94, "ppl": 1.00803, "step": 296, "tokens/total": 9645600, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 135321} +{"epoch": 1.16015625, "grad_norm": 0.28311464190483093, "learning_rate": 4.705880619288153e-05, "loss": 0.0005465521826408803, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00055, "step": 297, "tokens/total": 9679456, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 135765} +{"epoch": 1.1640625, "grad_norm": 0.03163145110011101, "learning_rate": 4.677323837567412e-05, "loss": 0.0001329109800281003, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00013, "step": 298, "tokens/total": 9711696, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 136196} +{"epoch": 1.16796875, "grad_norm": 0.0973736122250557, "learning_rate": 4.6488012907598146e-05, "loss": 0.00028265794389881194, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00028, "step": 299, "tokens/total": 9745792, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 136649} +{"epoch": 1.171875, "grad_norm": 0.5288437604904175, "learning_rate": 4.620314165804964e-05, "loss": 0.0074498518370091915, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00748, "step": 300, "tokens/total": 9780648, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 137128} +{"epoch": 1.17578125, "grad_norm": 0.07140754908323288, "learning_rate": 4.591863648168407e-05, "loss": 0.0005951586063019931, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.0006, "step": 301, "tokens/total": 9813032, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 137585} +{"epoch": 1.1796875, "grad_norm": 0.1837845742702484, "learning_rate": 4.5634509217923135e-05, "loss": 0.0019731561187654734, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00198, "step": 302, "tokens/total": 9845200, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 138054} +{"epoch": 1.18359375, "grad_norm": 0.21797706186771393, "learning_rate": 4.535077169046201e-05, "loss": 0.00125863344874233, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.29, "memory/max_allocated (GiB)": 65.29, "ppl": 1.00126, "step": 303, "tokens/total": 9878616, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 138559} +{"epoch": 1.1875, "grad_norm": 0.16737471520900726, "learning_rate": 4.506743570677743e-05, "loss": 0.0031781138386577368, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00318, "step": 304, "tokens/total": 9910776, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 139050} +{"epoch": 1.19140625, "grad_norm": 0.008498752489686012, "learning_rate": 4.478451305763618e-05, "loss": 0.00017386028775945306, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00017, "step": 305, "tokens/total": 9945480, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 139525} +{"epoch": 1.1953125, "grad_norm": 0.014850846491754055, "learning_rate": 4.450201551660454e-05, "loss": 0.00022009828535374254, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.00022, "step": 306, "tokens/total": 9975992, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 139969} +{"epoch": 1.19921875, "grad_norm": 0.27440837025642395, "learning_rate": 4.4219954839558276e-05, "loss": 0.0026099635288119316, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.00261, "step": 307, "tokens/total": 10006440, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 140443} +{"epoch": 1.203125, "grad_norm": 0.02934010699391365, "learning_rate": 4.393834276419352e-05, "loss": 0.0003221653460059315, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00032, "step": 308, "tokens/total": 10042928, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 140926} +{"epoch": 1.20703125, "grad_norm": 0.0060567064210772514, "learning_rate": 4.36571910095382e-05, "loss": 9.969036909751594e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.0001, "step": 309, "tokens/total": 10075184, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 141344} +{"epoch": 1.2109375, "grad_norm": 0.16351395845413208, "learning_rate": 4.337651127546448e-05, "loss": 0.0024599884636700153, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00246, "step": 310, "tokens/total": 10106656, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 141753} +{"epoch": 1.21484375, "grad_norm": 0.20014341175556183, "learning_rate": 4.3096315242201736e-05, "loss": 0.0010304638417437673, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.82, "memory/max_allocated (GiB)": 63.82, "ppl": 1.00103, "step": 311, "tokens/total": 10138536, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 142219} +{"epoch": 1.21875, "grad_norm": 0.16542315483093262, "learning_rate": 4.2816614569850635e-05, "loss": 0.002143880585208535, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.00215, "step": 312, "tokens/total": 10165768, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 142617} +{"epoch": 1.22265625, "grad_norm": 0.14379875361919403, "learning_rate": 4.2537420897897864e-05, "loss": 0.0009447492193430662, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00095, "step": 313, "tokens/total": 10201624, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 143103} +{"epoch": 1.2265625, "grad_norm": 0.13214077055454254, "learning_rate": 4.225874584473174e-05, "loss": 0.001281393226236105, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00128, "step": 314, "tokens/total": 10233304, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 143535} +{"epoch": 1.23046875, "grad_norm": 0.03736739978194237, "learning_rate": 4.19806010071587e-05, "loss": 0.00026818583137355745, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00027, "step": 315, "tokens/total": 10267904, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 144009} +{"epoch": 1.234375, "grad_norm": 0.738044798374176, "learning_rate": 4.170299795992081e-05, "loss": 0.00825062207877636, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00828, "step": 316, "tokens/total": 10301280, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 144460} +{"epoch": 1.23828125, "grad_norm": 0.19988492131233215, "learning_rate": 4.142594825521398e-05, "loss": 0.0026203899178653955, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00262, "step": 317, "tokens/total": 10337072, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 144936} +{"epoch": 1.2421875, "grad_norm": 0.027803786098957062, "learning_rate": 4.114946342220728e-05, "loss": 0.0002841686364263296, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.69, "memory/max_allocated (GiB)": 64.69, "ppl": 1.00028, "step": 318, "tokens/total": 10368688, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 145375} +{"epoch": 1.24609375, "grad_norm": 0.15930841863155365, "learning_rate": 4.087355496656321e-05, "loss": 0.001972435973584652, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00197, "step": 319, "tokens/total": 10400752, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 145849} +{"epoch": 1.25, "grad_norm": 0.27973759174346924, "learning_rate": 4.05982343699588e-05, "loss": 0.0023663192987442017, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00237, "step": 320, "tokens/total": 10432112, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 146318} +{"epoch": 1.25390625, "grad_norm": 0.06596971303224564, "learning_rate": 4.0323513089607876e-05, "loss": 0.00027560340822674334, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00028, "step": 321, "tokens/total": 10465472, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 146821} +{"epoch": 1.2578125, "grad_norm": 0.012164449319243431, "learning_rate": 4.004940255778431e-05, "loss": 8.907356823328882e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00009, "step": 322, "tokens/total": 10496336, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 147285} +{"epoch": 1.26171875, "grad_norm": 0.010241443291306496, "learning_rate": 3.977591418134619e-05, "loss": 7.19298332114704e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00007, "step": 323, "tokens/total": 10525864, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 147729} +{"epoch": 1.265625, "grad_norm": 0.8626437187194824, "learning_rate": 3.95030593412612e-05, "loss": 0.022279435768723488, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.02253, "step": 324, "tokens/total": 10559680, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 148169} +{"epoch": 1.26953125, "grad_norm": 0.037944111973047256, "learning_rate": 3.923084939213296e-05, "loss": 0.0002747695834841579, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00027, "step": 325, "tokens/total": 10595400, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 148630} +{"epoch": 1.2734375, "grad_norm": 0.2687492370605469, "learning_rate": 3.895929566172861e-05, "loss": 0.00745359156280756, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00748, "step": 326, "tokens/total": 10626680, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 149063} +{"epoch": 1.27734375, "grad_norm": 0.046988748013973236, "learning_rate": 3.868840945050728e-05, "loss": 0.0007325682090595365, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00073, "step": 327, "tokens/total": 10661784, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 149549} +{"epoch": 1.28125, "grad_norm": 0.5682485699653625, "learning_rate": 3.841820203114995e-05, "loss": 0.021844979375600815, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.31, "memory/max_allocated (GiB)": 65.31, "ppl": 1.02209, "step": 328, "tokens/total": 10696184, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 150015} +{"epoch": 1.28515625, "grad_norm": 0.01807788573205471, "learning_rate": 3.814868464809027e-05, "loss": 7.860039477236569e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00008, "step": 329, "tokens/total": 10729376, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 150514} +{"epoch": 1.2890625, "grad_norm": 0.05275453254580498, "learning_rate": 3.787986851704667e-05, "loss": 0.00020131940254941583, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.0002, "step": 330, "tokens/total": 10764824, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 151021} +{"epoch": 1.29296875, "grad_norm": 0.5170391798019409, "learning_rate": 3.7611764824555654e-05, "loss": 0.0044119395315647125, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00442, "step": 331, "tokens/total": 10797072, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 151486} +{"epoch": 1.296875, "grad_norm": 0.4565140902996063, "learning_rate": 3.734438472750619e-05, "loss": 0.011398390866816044, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.01146, "step": 332, "tokens/total": 10829376, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 151956} +{"epoch": 1.30078125, "grad_norm": 0.015961941331624985, "learning_rate": 3.707773935267552e-05, "loss": 0.0002222329640062526, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00022, "step": 333, "tokens/total": 10863512, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 152442} +{"epoch": 1.3046875, "grad_norm": 0.04985252022743225, "learning_rate": 3.68118397962661e-05, "loss": 0.00043848584755323827, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00044, "step": 334, "tokens/total": 10894432, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 152890} +{"epoch": 1.30859375, "grad_norm": 0.36925753951072693, "learning_rate": 3.654669712344384e-05, "loss": 0.008220167830586433, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.34, "memory/max_allocated (GiB)": 63.34, "ppl": 1.00825, "step": 335, "tokens/total": 10924848, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 153368} +{"epoch": 1.3125, "grad_norm": 0.548951268196106, "learning_rate": 3.628232236787763e-05, "loss": 0.0045019518584012985, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00451, "step": 336, "tokens/total": 10959024, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 153809} +{"epoch": 1.31640625, "grad_norm": 0.6940018534660339, "learning_rate": 3.6018726531280144e-05, "loss": 0.015694651752710342, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.01582, "step": 337, "tokens/total": 10991088, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 154258} +{"epoch": 1.3203125, "grad_norm": 0.2448171079158783, "learning_rate": 3.575592058295017e-05, "loss": 0.003911243751645088, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00392, "step": 338, "tokens/total": 11023904, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 154704} +{"epoch": 1.32421875, "grad_norm": 0.021085431799292564, "learning_rate": 3.549391545931585e-05, "loss": 0.0003873141831718385, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00039, "step": 339, "tokens/total": 11057680, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 155157} +{"epoch": 1.328125, "grad_norm": 0.05811301991343498, "learning_rate": 3.5232722063479914e-05, "loss": 0.001011216314509511, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00101, "step": 340, "tokens/total": 11089800, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 155582} +{"epoch": 1.33203125, "grad_norm": 0.13535913825035095, "learning_rate": 3.49723512647657e-05, "loss": 0.0009759629028849304, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00098, "step": 341, "tokens/total": 11117960, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 156047} +{"epoch": 1.3359375, "grad_norm": 0.1454484760761261, "learning_rate": 3.471281389826491e-05, "loss": 0.002040302846580744, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00204, "step": 342, "tokens/total": 11151888, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 156500} +{"epoch": 1.33984375, "grad_norm": 0.0766342431306839, "learning_rate": 3.4454120764386764e-05, "loss": 0.0010450899135321379, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.22, "memory/max_allocated (GiB)": 65.22, "ppl": 1.00105, "step": 343, "tokens/total": 11184568, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 156931} +{"epoch": 1.34375, "grad_norm": 0.3910510838031769, "learning_rate": 3.4196282628408526e-05, "loss": 0.008200807496905327, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00823, "step": 344, "tokens/total": 11216400, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 157385} +{"epoch": 1.34765625, "grad_norm": 0.019846156239509583, "learning_rate": 3.3939310220027456e-05, "loss": 0.00036623989581130445, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00037, "step": 345, "tokens/total": 11250768, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 157817} +{"epoch": 1.3515625, "grad_norm": 0.09740956127643585, "learning_rate": 3.3683214232914404e-05, "loss": 0.0009770817123353481, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00098, "step": 346, "tokens/total": 11281808, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 158273} +{"epoch": 1.35546875, "grad_norm": 0.041435085237026215, "learning_rate": 3.342800532426873e-05, "loss": 0.0010718714911490679, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00107, "step": 347, "tokens/total": 11313968, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 158728} +{"epoch": 1.359375, "grad_norm": 0.008635983802378178, "learning_rate": 3.317369411437484e-05, "loss": 0.0001968650904018432, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.0002, "step": 348, "tokens/total": 11347288, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 159168} +{"epoch": 1.36328125, "grad_norm": 0.2318911850452423, "learning_rate": 3.292029118616024e-05, "loss": 0.004757854156196117, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00477, "step": 349, "tokens/total": 11377128, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 159592} +{"epoch": 1.3671875, "grad_norm": 0.04282344877719879, "learning_rate": 3.266780708475511e-05, "loss": 0.0006862574373371899, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00069, "step": 350, "tokens/total": 11409488, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 160093} +{"epoch": 1.37109375, "grad_norm": 0.05374312773346901, "learning_rate": 3.241625231705354e-05, "loss": 0.0007393195992335677, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00074, "step": 351, "tokens/total": 11442040, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 160534} +{"epoch": 1.375, "grad_norm": 0.15490907430648804, "learning_rate": 3.216563735127618e-05, "loss": 0.002007519593462348, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00201, "step": 352, "tokens/total": 11478312, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 161009} +{"epoch": 1.37890625, "grad_norm": 0.1076049879193306, "learning_rate": 3.191597261653475e-05, "loss": 0.002416109899058938, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.93, "memory/max_allocated (GiB)": 63.93, "ppl": 1.00242, "step": 353, "tokens/total": 11509008, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 161450} +{"epoch": 1.3828125, "grad_norm": 0.3269493579864502, "learning_rate": 3.166726850239794e-05, "loss": 0.008129325695335865, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00816, "step": 354, "tokens/total": 11541264, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 161900} +{"epoch": 1.38671875, "grad_norm": 0.0060245562344789505, "learning_rate": 3.141953535845912e-05, "loss": 0.00013305083848536015, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00013, "step": 355, "tokens/total": 11573072, "tokens/train_per_sec_per_gpu": 10.01, "tokens/trainable": 162358} +{"epoch": 1.390625, "grad_norm": 0.1991882175207138, "learning_rate": 3.11727834939056e-05, "loss": 0.0021785127464681864, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00218, "step": 356, "tokens/total": 11605224, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 162845} +{"epoch": 1.39453125, "grad_norm": 0.3355052173137665, "learning_rate": 3.092702317708967e-05, "loss": 0.0041597033850848675, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00417, "step": 357, "tokens/total": 11637512, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 163299} +{"epoch": 1.3984375, "grad_norm": 0.15223896503448486, "learning_rate": 3.0682264635101276e-05, "loss": 0.0034436937421560287, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00345, "step": 358, "tokens/total": 11671280, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 163774} +{"epoch": 1.40234375, "grad_norm": 0.29581740498542786, "learning_rate": 3.0438518053342407e-05, "loss": 0.011507068760693073, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.01157, "step": 359, "tokens/total": 11702008, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 164199} +{"epoch": 1.40625, "grad_norm": 0.1360633671283722, "learning_rate": 3.0195793575103266e-05, "loss": 0.0016909398837015033, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00169, "step": 360, "tokens/total": 11732032, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 164667} +{"epoch": 1.41015625, "grad_norm": 0.24053215980529785, "learning_rate": 2.9954101301140146e-05, "loss": 0.008216691203415394, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00825, "step": 361, "tokens/total": 11767472, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.01405306626111269, "learning_rate": 2.9713451289255123e-05, "loss": 0.0002941336133517325, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00029, "step": 362, "tokens/total": 11798432, "tokens/train_per_sec_per_gpu": 10.39, "tokens/trainable": 165575} +{"epoch": 1.41796875, "grad_norm": 0.2097272127866745, "learning_rate": 2.9473853553877484e-05, "loss": 0.0011080435942858458, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00111, "step": 363, "tokens/total": 11833144, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 166060} +{"epoch": 1.421875, "grad_norm": 0.02843608893454075, "learning_rate": 2.9235318065647e-05, "loss": 0.00028309665503911674, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.00028, "step": 364, "tokens/total": 11865856, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 166478} +{"epoch": 1.42578125, "grad_norm": 1.6293288469314575, "learning_rate": 2.8997854750998964e-05, "loss": 0.01932467892765999, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.01951, "step": 365, "tokens/total": 11900400, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 166941} +{"epoch": 1.4296875, "grad_norm": 0.06661222130060196, "learning_rate": 2.8761473491751258e-05, "loss": 0.001044339733198285, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.49, "memory/max_allocated (GiB)": 63.49, "ppl": 1.00104, "step": 366, "tokens/total": 11931368, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 167381} +{"epoch": 1.43359375, "grad_norm": 0.4945783019065857, "learning_rate": 2.8526184124692883e-05, "loss": 0.0027549804653972387, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00276, "step": 367, "tokens/total": 11963320, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 167830} +{"epoch": 1.4375, "grad_norm": 0.013173859566450119, "learning_rate": 2.829199644117484e-05, "loss": 0.00018174099386669695, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00018, "step": 368, "tokens/total": 11997536, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 168266} +{"epoch": 1.44140625, "grad_norm": 0.03221866115927696, "learning_rate": 2.8058920186702553e-05, "loss": 0.00026882061501964927, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00027, "step": 369, "tokens/total": 12029528, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 168716} +{"epoch": 1.4453125, "grad_norm": 0.028212325647473335, "learning_rate": 2.782696506053033e-05, "loss": 0.0004042961518280208, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.0004, "step": 370, "tokens/total": 12062504, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 169176} +{"epoch": 1.44921875, "grad_norm": 0.09399929642677307, "learning_rate": 2.7596140715257824e-05, "loss": 0.0014715055003762245, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.78, "memory/max_allocated (GiB)": 65.78, "ppl": 1.00147, "step": 371, "tokens/total": 12097376, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 169591} +{"epoch": 1.453125, "grad_norm": 0.03815427049994469, "learning_rate": 2.7366456756428184e-05, "loss": 0.0004917392507195473, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00049, "step": 372, "tokens/total": 12131920, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 170041} +{"epoch": 1.45703125, "grad_norm": 0.26894906163215637, "learning_rate": 2.7137922742128486e-05, "loss": 0.0035483527462929487, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00355, "step": 373, "tokens/total": 12164528, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 170516} +{"epoch": 1.4609375, "grad_norm": 0.05754311382770538, "learning_rate": 2.691054818259188e-05, "loss": 0.0010004672221839428, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.001, "step": 374, "tokens/total": 12197376, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 170952} +{"epoch": 1.46484375, "grad_norm": 0.04939396306872368, "learning_rate": 2.6684342539801933e-05, "loss": 0.00036297852057032287, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00036, "step": 375, "tokens/total": 12233104, "tokens/train_per_sec_per_gpu": 7.12, "tokens/trainable": 171424} +{"epoch": 1.46875, "grad_norm": 0.004489637911319733, "learning_rate": 2.645931522709877e-05, "loss": 7.183748675743118e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.69, "memory/max_allocated (GiB)": 65.69, "ppl": 1.00007, "step": 376, "tokens/total": 12269224, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 171884} +{"epoch": 1.47265625, "grad_norm": 0.03292468190193176, "learning_rate": 2.6235475608787365e-05, "loss": 0.00035930919693782926, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00036, "step": 377, "tokens/total": 12302952, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 172404} +{"epoch": 1.4765625, "grad_norm": 0.03538944944739342, "learning_rate": 2.6012832999747916e-05, "loss": 0.00039594151894561946, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.0004, "step": 378, "tokens/total": 12338424, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 172871} +{"epoch": 1.48046875, "grad_norm": 0.2977791130542755, "learning_rate": 2.579139666504821e-05, "loss": 0.0024246901739388704, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00243, "step": 379, "tokens/total": 12369144, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 173329} +{"epoch": 1.484375, "grad_norm": 0.14272575080394745, "learning_rate": 2.557117581955798e-05, "loss": 0.0010031553683802485, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.001, "step": 380, "tokens/total": 12403352, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 173827} +{"epoch": 1.48828125, "grad_norm": 0.02246430329978466, "learning_rate": 2.5352179627565532e-05, "loss": 0.00040720216929912567, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00041, "step": 381, "tokens/total": 12441056, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 174287} +{"epoch": 1.4921875, "grad_norm": 0.059240929782390594, "learning_rate": 2.5134417202396277e-05, "loss": 0.0006545068463310599, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.00065, "step": 382, "tokens/total": 12474080, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 174731} +{"epoch": 1.49609375, "grad_norm": 0.01776772364974022, "learning_rate": 2.491789760603361e-05, "loss": 0.00036312444717623293, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00036, "step": 383, "tokens/total": 12508992, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 175183} +{"epoch": 1.5, "grad_norm": 0.010496149770915508, "learning_rate": 2.4702629848741764e-05, "loss": 0.0001560437958687544, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00016, "step": 384, "tokens/total": 12540896, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 175655} +{"epoch": 1.50390625, "grad_norm": 0.01213150005787611, "learning_rate": 2.4488622888690785e-05, "loss": 0.00018711121811065823, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00019, "step": 385, "tokens/total": 12568416, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 176085} +{"epoch": 1.5078125, "grad_norm": 0.17884396016597748, "learning_rate": 2.427588563158384e-05, "loss": 0.0010500064818188548, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00105, "step": 386, "tokens/total": 12599792, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 176549} +{"epoch": 1.51171875, "grad_norm": 0.08864817023277283, "learning_rate": 2.406442693028651e-05, "loss": 0.0008040435495786369, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0008, "step": 387, "tokens/total": 12631224, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 176995} +{"epoch": 1.515625, "grad_norm": 0.14845100045204163, "learning_rate": 2.3854255584458547e-05, "loss": 0.0005871801404282451, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00059, "step": 388, "tokens/total": 12667240, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 177468} +{"epoch": 1.51953125, "grad_norm": 0.023912567645311356, "learning_rate": 2.3645380340187508e-05, "loss": 0.00028977676993235946, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.54, "memory/max_allocated (GiB)": 65.54, "ppl": 1.00029, "step": 389, "tokens/total": 12700488, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 177961} +{"epoch": 1.5234375, "grad_norm": 0.005154292564839125, "learning_rate": 2.3437809889624914e-05, "loss": 7.515643665101379e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.00008, "step": 390, "tokens/total": 12734528, "tokens/train_per_sec_per_gpu": 6.91, "tokens/trainable": 178417} +{"epoch": 1.52734375, "grad_norm": 0.009530507028102875, "learning_rate": 2.3231552870624487e-05, "loss": 0.00012624952069018036, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00013, "step": 391, "tokens/total": 12767152, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 178862} +{"epoch": 1.53125, "grad_norm": 0.08778093755245209, "learning_rate": 2.3026617866382657e-05, "loss": 0.0005683842464350164, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.00057, "step": 392, "tokens/total": 12798168, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 179320} +{"epoch": 1.53515625, "grad_norm": 0.09487402439117432, "learning_rate": 2.2823013405081507e-05, "loss": 0.0006001405417919159, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.0006, "step": 393, "tokens/total": 12828896, "tokens/train_per_sec_per_gpu": 9.81, "tokens/trainable": 179758} +{"epoch": 1.5390625, "grad_norm": 0.2523862421512604, "learning_rate": 2.2620747959533722e-05, "loss": 0.002670936519280076, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00267, "step": 394, "tokens/total": 12860024, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 180211} +{"epoch": 1.54296875, "grad_norm": 0.7581096887588501, "learning_rate": 2.2419829946830123e-05, "loss": 0.011065435595810413, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01113, "step": 395, "tokens/total": 12891472, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 180637} +{"epoch": 1.546875, "grad_norm": 0.17290620505809784, "learning_rate": 2.2220267727989325e-05, "loss": 0.0013716385001316667, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00137, "step": 396, "tokens/total": 12925648, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 181142} +{"epoch": 1.55078125, "grad_norm": 0.013067585416138172, "learning_rate": 2.202206960760984e-05, "loss": 0.00020362346549518406, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.77, "memory/max_allocated (GiB)": 64.77, "ppl": 1.0002, "step": 397, "tokens/total": 12959200, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 181597} +{"epoch": 1.5546875, "grad_norm": 0.055604640394449234, "learning_rate": 2.182524383352446e-05, "loss": 0.00063690374372527, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00064, "step": 398, "tokens/total": 12989624, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 182030} +{"epoch": 1.55859375, "grad_norm": 0.0014103141147643328, "learning_rate": 2.1629798596457056e-05, "loss": 2.673912422324065e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 62.97, "memory/max_allocated (GiB)": 62.97, "ppl": 1.00003, "step": 399, "tokens/total": 13019040, "tokens/train_per_sec_per_gpu": 10.23, "tokens/trainable": 182496} +{"epoch": 1.5625, "grad_norm": 0.22677931189537048, "learning_rate": 2.1435742029681725e-05, "loss": 0.0004980148514732718, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.0005, "step": 400, "tokens/total": 13051808, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 182963} +{"epoch": 1.56640625, "grad_norm": 0.012545973993837833, "learning_rate": 2.124308220868431e-05, "loss": 0.00015173523570410907, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00015, "step": 401, "tokens/total": 13085520, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 183416} +{"epoch": 1.5703125, "grad_norm": 0.0018952707760035992, "learning_rate": 2.105182715082638e-05, "loss": 2.9560967959696427e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00003, "step": 402, "tokens/total": 13116224, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 183857} +{"epoch": 1.57421875, "grad_norm": 0.01129449624568224, "learning_rate": 2.0861984815011552e-05, "loss": 6.552524428116158e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00007, "step": 403, "tokens/total": 13149984, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 184296} +{"epoch": 1.578125, "grad_norm": 0.06993789225816727, "learning_rate": 2.0673563101354323e-05, "loss": 0.0003621559008024633, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00036, "step": 404, "tokens/total": 13185488, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 184719} +{"epoch": 1.58203125, "grad_norm": 0.008687548339366913, "learning_rate": 2.0486569850851317e-05, "loss": 9.778824460227042e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.0001, "step": 405, "tokens/total": 13216320, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 185172} +{"epoch": 1.5859375, "grad_norm": 0.004937805701047182, "learning_rate": 2.0301012845054956e-05, "loss": 4.1570609027985483e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00004, "step": 406, "tokens/total": 13248136, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 185646} +{"epoch": 1.58984375, "grad_norm": 0.03992395102977753, "learning_rate": 2.011689980574966e-05, "loss": 0.00033147953217849135, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00033, "step": 407, "tokens/total": 13280640, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 186129} +{"epoch": 1.59375, "grad_norm": 0.048533808439970016, "learning_rate": 1.993423839463052e-05, "loss": 0.0004494082531891763, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00045, "step": 408, "tokens/total": 13314160, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 186610} +{"epoch": 1.59765625, "grad_norm": 0.07362967729568481, "learning_rate": 1.975303621298445e-05, "loss": 0.0007629890460520983, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.51, "memory/max_allocated (GiB)": 63.51, "ppl": 1.00076, "step": 409, "tokens/total": 13345024, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 187066} +{"epoch": 1.6015625, "grad_norm": 0.002379094948992133, "learning_rate": 1.957330080137385e-05, "loss": 2.8739414119627327e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00003, "step": 410, "tokens/total": 13377248, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 187567} +{"epoch": 1.60546875, "grad_norm": 0.0029808427207171917, "learning_rate": 1.9395039639322864e-05, "loss": 4.0978062315844e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00004, "step": 411, "tokens/total": 13410992, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 188056} +{"epoch": 1.609375, "grad_norm": 0.02669011801481247, "learning_rate": 1.9218260145006073e-05, "loss": 0.0002692687849048525, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00027, "step": 412, "tokens/total": 13442728, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 188502} +{"epoch": 1.61328125, "grad_norm": 0.0104796988889575, "learning_rate": 1.904296967493982e-05, "loss": 0.00010727354674600065, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00011, "step": 413, "tokens/total": 13474824, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 188995} +{"epoch": 1.6171875, "grad_norm": 0.5167486071586609, "learning_rate": 1.8869175523676064e-05, "loss": 0.003539609955623746, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00355, "step": 414, "tokens/total": 13506528, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 189483} +{"epoch": 1.62109375, "grad_norm": 0.24249520897865295, "learning_rate": 1.869688492349885e-05, "loss": 0.0019101985963061452, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00191, "step": 415, "tokens/total": 13537792, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 189935} +{"epoch": 1.625, "grad_norm": 0.009620090946555138, "learning_rate": 1.85261050441233e-05, "loss": 9.288715955335647e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.00009, "step": 416, "tokens/total": 13573136, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 190399} +{"epoch": 1.62890625, "grad_norm": 0.00463481992483139, "learning_rate": 1.8356842992397304e-05, "loss": 7.441082561854273e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00007, "step": 417, "tokens/total": 13607240, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 190862} +{"epoch": 1.6328125, "grad_norm": 0.00024082895833998919, "learning_rate": 1.8189105812005714e-05, "loss": 4.605084086506395e-06, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.0, "step": 418, "tokens/total": 13638920, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 191294} +{"epoch": 1.63671875, "grad_norm": 0.001834060181863606, "learning_rate": 1.802290048317732e-05, "loss": 2.713160210987553e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00003, "step": 419, "tokens/total": 13672320, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 191764} +{"epoch": 1.640625, "grad_norm": 0.02224600687623024, "learning_rate": 1.785823392239424e-05, "loss": 0.00023223830794449896, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.00023, "step": 420, "tokens/total": 13703312, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 192196} +{"epoch": 1.64453125, "grad_norm": 0.6952553987503052, "learning_rate": 1.7695112982104225e-05, "loss": 0.010719376616179943, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.01078, "step": 421, "tokens/total": 13734440, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 192612} +{"epoch": 1.6484375, "grad_norm": 0.0013984201941639185, "learning_rate": 1.7533544450435433e-05, "loss": 1.0299647328793071e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.62, "memory/max_allocated (GiB)": 64.62, "ppl": 1.00001, "step": 422, "tokens/total": 13766944, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 193059} +{"epoch": 1.65234375, "grad_norm": 1.3604685068130493, "learning_rate": 1.7373535050913946e-05, "loss": 0.013203768990933895, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.01329, "step": 423, "tokens/total": 13797120, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 193466} +{"epoch": 1.65625, "grad_norm": 0.39403921365737915, "learning_rate": 1.721509144218405e-05, "loss": 0.0018176654120907187, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00182, "step": 424, "tokens/total": 13828040, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 193910} +{"epoch": 1.66015625, "grad_norm": 0.0028262659907341003, "learning_rate": 1.705822021773101e-05, "loss": 1.8616254237713292e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.06, "memory/max_allocated (GiB)": 63.06, "ppl": 1.00002, "step": 425, "tokens/total": 13856328, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 194314} +{"epoch": 1.6640625, "grad_norm": 1.0897696018218994, "learning_rate": 1.69029279056068e-05, "loss": 0.016455812379717827, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01659, "step": 426, "tokens/total": 13891312, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 194782} +{"epoch": 1.66796875, "grad_norm": 0.4920905530452728, "learning_rate": 1.6749220968158415e-05, "loss": 0.013386040925979614, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.01348, "step": 427, "tokens/total": 13925272, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 195254} +{"epoch": 1.671875, "grad_norm": 0.9280214905738831, "learning_rate": 1.659710580175893e-05, "loss": 0.01056643109768629, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.53, "memory/max_allocated (GiB)": 63.53, "ppl": 1.01062, "step": 428, "tokens/total": 13955752, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 195702} +{"epoch": 1.67578125, "grad_norm": 0.24581065773963928, "learning_rate": 1.644658873654133e-05, "loss": 0.0036977282725274563, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.0037, "step": 429, "tokens/total": 13988000, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 196164} +{"epoch": 1.6796875, "grad_norm": 0.11312751471996307, "learning_rate": 1.629767603613508e-05, "loss": 0.0007814400596544147, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00078, "step": 430, "tokens/total": 14018712, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 196604} +{"epoch": 1.68359375, "grad_norm": 0.09665724635124207, "learning_rate": 1.615037389740547e-05, "loss": 0.0010589923476800323, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00106, "step": 431, "tokens/total": 14051992, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 197070} +{"epoch": 1.6875, "grad_norm": 0.004201829899102449, "learning_rate": 1.600468845019576e-05, "loss": 5.6724646128714085e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.36, "memory/max_allocated (GiB)": 63.36, "ppl": 1.00006, "step": 432, "tokens/total": 14082624, "tokens/train_per_sec_per_gpu": 10.31, "tokens/trainable": 197552} +{"epoch": 1.69140625, "grad_norm": 0.038906194269657135, "learning_rate": 1.5860625757072092e-05, "loss": 0.0003083007759414613, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00031, "step": 433, "tokens/total": 14113304, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 197990} +{"epoch": 1.6953125, "grad_norm": 0.02319645695388317, "learning_rate": 1.571819181307116e-05, "loss": 0.00020796216267626733, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00021, "step": 434, "tokens/total": 14149592, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 198513} +{"epoch": 1.69921875, "grad_norm": 0.01930493302643299, "learning_rate": 1.557739254545075e-05, "loss": 0.00030133500695228577, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.0003, "step": 435, "tokens/total": 14183584, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 198979} +{"epoch": 1.703125, "grad_norm": 0.0073189144022762775, "learning_rate": 1.543823381344311e-05, "loss": 8.07775286375545e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00008, "step": 436, "tokens/total": 14213688, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 199443} +{"epoch": 1.70703125, "grad_norm": 0.08134113252162933, "learning_rate": 1.5300721408011114e-05, "loss": 0.0005738002364523709, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.63, "memory/max_allocated (GiB)": 65.63, "ppl": 1.00057, "step": 437, "tokens/total": 14246480, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 199899} +{"epoch": 1.7109375, "grad_norm": 0.18651598691940308, "learning_rate": 1.5164861051607254e-05, "loss": 0.0007677439716644585, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00077, "step": 438, "tokens/total": 14278320, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 200368} +{"epoch": 1.71484375, "grad_norm": 0.5389974117279053, "learning_rate": 1.5030658397935521e-05, "loss": 0.005872917361557484, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00589, "step": 439, "tokens/total": 14313016, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 200819} +{"epoch": 1.71875, "grad_norm": 0.009340011514723301, "learning_rate": 1.4898119031716104e-05, "loss": 0.00013479507470037788, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00013, "step": 440, "tokens/total": 14343352, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 201275} +{"epoch": 1.72265625, "grad_norm": 0.02824859507381916, "learning_rate": 1.476724846845306e-05, "loss": 0.0002614363038446754, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00026, "step": 441, "tokens/total": 14377736, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 201699} +{"epoch": 1.7265625, "grad_norm": 0.12170938402414322, "learning_rate": 1.463805215420471e-05, "loss": 0.0011410234728828073, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00114, "step": 442, "tokens/total": 14415056, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 202200} +{"epoch": 1.73046875, "grad_norm": 0.09817881882190704, "learning_rate": 1.451053546535705e-05, "loss": 0.0011835386976599693, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00118, "step": 443, "tokens/total": 14449544, "tokens/train_per_sec_per_gpu": 6.92, "tokens/trainable": 202640} +{"epoch": 1.734375, "grad_norm": 0.010307974182069302, "learning_rate": 1.438470370840001e-05, "loss": 0.0001215864103869535, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00012, "step": 444, "tokens/total": 14483472, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 203122} +{"epoch": 1.73828125, "grad_norm": 0.03886914625763893, "learning_rate": 1.4260562119706606e-05, "loss": 0.00035674989339895546, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.19, "memory/max_allocated (GiB)": 65.19, "ppl": 1.00036, "step": 445, "tokens/total": 14517648, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 203587} +{"epoch": 1.7421875, "grad_norm": 0.004266851581633091, "learning_rate": 1.413811586531508e-05, "loss": 9.611135465092957e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.0001, "step": 446, "tokens/total": 14553056, "tokens/train_per_sec_per_gpu": 6.72, "tokens/trainable": 204024} +{"epoch": 1.74609375, "grad_norm": 0.014354717917740345, "learning_rate": 1.4017370040713884e-05, "loss": 0.0001816036383388564, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.00018, "step": 447, "tokens/total": 14584280, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 204465} +{"epoch": 1.75, "grad_norm": 0.31285175681114197, "learning_rate": 1.3898329670629645e-05, "loss": 0.0030830700416117907, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00309, "step": 448, "tokens/total": 14618320, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 204923} +{"epoch": 1.75390625, "grad_norm": 0.011388896033167839, "learning_rate": 1.3780999708818058e-05, "loss": 0.00019246367446612567, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00019, "step": 449, "tokens/total": 14651264, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 205388} +{"epoch": 1.7578125, "grad_norm": 0.05953628942370415, "learning_rate": 1.3665385037857758e-05, "loss": 0.00018775051285047084, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00019, "step": 450, "tokens/total": 14682896, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 205873} +{"epoch": 1.76171875, "grad_norm": 0.1080264300107956, "learning_rate": 1.3551490468947126e-05, "loss": 0.0007755247643217444, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00078, "step": 451, "tokens/total": 14715272, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 206346} +{"epoch": 1.765625, "grad_norm": 0.5192931294441223, "learning_rate": 1.3439320741704075e-05, "loss": 0.0026660964358597994, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00267, "step": 452, "tokens/total": 14744840, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 206776} +{"epoch": 1.76953125, "grad_norm": 0.00904427096247673, "learning_rate": 1.3328880523968808e-05, "loss": 0.000198873138288036, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.0002, "step": 453, "tokens/total": 14778008, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 207233} +{"epoch": 1.7734375, "grad_norm": 0.0713205486536026, "learning_rate": 1.3220174411609587e-05, "loss": 0.0004167373990640044, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00042, "step": 454, "tokens/total": 14808472, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 207695} +{"epoch": 1.77734375, "grad_norm": 0.02103050984442234, "learning_rate": 1.3113206928331471e-05, "loss": 0.00021560602181125432, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00022, "step": 455, "tokens/total": 14841048, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 208144} +{"epoch": 1.78125, "grad_norm": 0.0732424259185791, "learning_rate": 1.300798252548806e-05, "loss": 0.0006713216425850987, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00067, "step": 456, "tokens/total": 14876904, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 208602} +{"epoch": 1.78515625, "grad_norm": 0.17008240520954132, "learning_rate": 1.2904505581896265e-05, "loss": 0.0009979375172406435, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.58, "memory/max_allocated (GiB)": 64.58, "ppl": 1.001, "step": 457, "tokens/total": 14909512, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 209040} +{"epoch": 1.7890625, "grad_norm": 0.18215201795101166, "learning_rate": 1.2802780403654082e-05, "loss": 0.001391434809193015, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00139, "step": 458, "tokens/total": 14942048, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 209501} +{"epoch": 1.79296875, "grad_norm": 0.3158450424671173, "learning_rate": 1.2702811223961408e-05, "loss": 0.004658947698771954, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.00467, "step": 459, "tokens/total": 14977032, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 209993} +{"epoch": 1.796875, "grad_norm": 0.28170695900917053, "learning_rate": 1.2604602202943861e-05, "loss": 0.004275145009160042, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00428, "step": 460, "tokens/total": 15011904, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 210455} +{"epoch": 1.80078125, "grad_norm": 0.5893635153770447, "learning_rate": 1.2508157427479686e-05, "loss": 0.011489270254969597, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.01156, "step": 461, "tokens/total": 15045504, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 210960} +{"epoch": 1.8046875, "grad_norm": 0.2678896486759186, "learning_rate": 1.2413480911029655e-05, "loss": 0.00228885724209249, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.15, "memory/max_allocated (GiB)": 66.15, "ppl": 1.00229, "step": 462, "tokens/total": 15078536, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 211405} +{"epoch": 1.80859375, "grad_norm": 0.17540624737739563, "learning_rate": 1.2320576593470082e-05, "loss": 0.0009677222114987671, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00097, "step": 463, "tokens/total": 15111640, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 211881} +{"epoch": 1.8125, "grad_norm": 0.019715407863259315, "learning_rate": 1.2229448340928828e-05, "loss": 0.000309724040562287, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00031, "step": 464, "tokens/total": 15142608, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 212338} +{"epoch": 1.81640625, "grad_norm": 0.08512537181377411, "learning_rate": 1.2140099945624458e-05, "loss": 0.0006418822449631989, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.00064, "step": 465, "tokens/total": 15175048, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 212803} +{"epoch": 1.8203125, "grad_norm": 0.013463757000863552, "learning_rate": 1.205253512570841e-05, "loss": 0.00021222344366833568, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00021, "step": 466, "tokens/total": 15205776, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 213215} +{"epoch": 1.82421875, "grad_norm": 0.1085970550775528, "learning_rate": 1.1966757525110255e-05, "loss": 0.0009141430491581559, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.00091, "step": 467, "tokens/total": 15239240, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 213675} +{"epoch": 1.828125, "grad_norm": 0.2816070020198822, "learning_rate": 1.1882770713386095e-05, "loss": 0.003010193817317486, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00301, "step": 468, "tokens/total": 15273696, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 214169} +{"epoch": 1.83203125, "grad_norm": 0.018265441060066223, "learning_rate": 1.180057818556998e-05, "loss": 0.00010770773224066943, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00011, "step": 469, "tokens/total": 15307632, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 214624} +{"epoch": 1.8359375, "grad_norm": 0.6393512487411499, "learning_rate": 1.1720183362028494e-05, "loss": 0.013407675549387932, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.0135, "step": 470, "tokens/total": 15339960, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 215087} +{"epoch": 1.83984375, "grad_norm": 0.005176741164177656, "learning_rate": 1.1641589588318387e-05, "loss": 0.00010164756531594321, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.0001, "step": 471, "tokens/total": 15373640, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 215553} +{"epoch": 1.84375, "grad_norm": 0.014590500853955746, "learning_rate": 1.1564800135047418e-05, "loss": 0.00020162445434834808, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.0002, "step": 472, "tokens/total": 15406304, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 216018} +{"epoch": 1.84765625, "grad_norm": 0.018243441358208656, "learning_rate": 1.148981819773816e-05, "loss": 0.0002203197800554335, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00022, "step": 473, "tokens/total": 15440128, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 216477} +{"epoch": 1.8515625, "grad_norm": 0.12023475021123886, "learning_rate": 1.1416646896695086e-05, "loss": 0.0013196554500609636, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00132, "step": 474, "tokens/total": 15471600, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 216949} +{"epoch": 1.85546875, "grad_norm": 0.021687136963009834, "learning_rate": 1.1345289276874717e-05, "loss": 0.00033237002207897604, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00033, "step": 475, "tokens/total": 15507016, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 217408} +{"epoch": 1.859375, "grad_norm": 0.25092723965644836, "learning_rate": 1.1275748307758873e-05, "loss": 0.004657765384763479, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00467, "step": 476, "tokens/total": 15542600, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 217854} +{"epoch": 1.86328125, "grad_norm": 0.021689455956220627, "learning_rate": 1.1208026883231147e-05, "loss": 0.000166562560480088, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00017, "step": 477, "tokens/total": 15577320, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 218337} +{"epoch": 1.8671875, "grad_norm": 0.12796518206596375, "learning_rate": 1.1142127821456433e-05, "loss": 0.0016470811096951365, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00165, "step": 478, "tokens/total": 15608840, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 218788} +{"epoch": 1.87109375, "grad_norm": 0.020341381430625916, "learning_rate": 1.1078053864763674e-05, "loss": 0.000207749311812222, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00021, "step": 479, "tokens/total": 15640160, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 219219} +{"epoch": 1.875, "grad_norm": 0.32333090901374817, "learning_rate": 1.1015807679531756e-05, "loss": 0.00125264679081738, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00125, "step": 480, "tokens/total": 15673920, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 219676} +{"epoch": 1.87890625, "grad_norm": 0.016019832342863083, "learning_rate": 1.0955391856078528e-05, "loss": 0.00013496669998858124, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00013, "step": 481, "tokens/total": 15705512, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 220125} +{"epoch": 1.8828125, "grad_norm": 0.01922600157558918, "learning_rate": 1.0896808908553007e-05, "loss": 0.00029498760704882443, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.0003, "step": 482, "tokens/total": 15742096, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 220615} +{"epoch": 1.88671875, "grad_norm": 0.8228985071182251, "learning_rate": 1.0840061274830763e-05, "loss": 0.0003269024600740522, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00033, "step": 483, "tokens/total": 15773064, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 221081} +{"epoch": 1.890625, "grad_norm": 0.293425977230072, "learning_rate": 1.0785151316412473e-05, "loss": 0.0017504653660580516, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00175, "step": 484, "tokens/total": 15805368, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 221546} +{"epoch": 1.89453125, "grad_norm": 0.014890726655721664, "learning_rate": 1.0732081318325639e-05, "loss": 0.00017020896484609693, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.57, "memory/max_allocated (GiB)": 65.57, "ppl": 1.00017, "step": 485, "tokens/total": 15839480, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 222014} +{"epoch": 1.8984375, "grad_norm": 0.12087839096784592, "learning_rate": 1.0680853489029501e-05, "loss": 0.00040791876381263137, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00041, "step": 486, "tokens/total": 15870672, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 222446} +{"epoch": 1.90234375, "grad_norm": 0.008386052213609219, "learning_rate": 1.0631469960323152e-05, "loss": 9.889474313240498e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.0001, "step": 487, "tokens/total": 15902784, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 222899} +{"epoch": 1.90625, "grad_norm": 0.24868960678577423, "learning_rate": 1.0583932787256783e-05, "loss": 0.001836068113334477, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00184, "step": 488, "tokens/total": 15935456, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 223333} +{"epoch": 1.91015625, "grad_norm": 0.007133211474865675, "learning_rate": 1.0538243948046206e-05, "loss": 5.827510904055089e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00006, "step": 489, "tokens/total": 15967960, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 223780} +{"epoch": 1.9140625, "grad_norm": 0.05028615519404411, "learning_rate": 1.0494405343990523e-05, "loss": 0.00042132235830649734, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00042, "step": 490, "tokens/total": 16003904, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 224253} +{"epoch": 1.91796875, "grad_norm": 0.01415669173002243, "learning_rate": 1.0452418799392985e-05, "loss": 0.00015647851978428662, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00016, "step": 491, "tokens/total": 16035816, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 224699} +{"epoch": 1.921875, "grad_norm": 0.025373809039592743, "learning_rate": 1.0412286061485102e-05, "loss": 0.0003404661547392607, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.26, "memory/max_allocated (GiB)": 65.26, "ppl": 1.00034, "step": 492, "tokens/total": 16066880, "tokens/train_per_sec_per_gpu": 10.35, "tokens/trainable": 225147} +{"epoch": 1.92578125, "grad_norm": 0.002072273287922144, "learning_rate": 1.03740088003539e-05, "loss": 3.0293958843685687e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00003, "step": 493, "tokens/total": 16101168, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 225596} +{"epoch": 1.9296875, "grad_norm": 0.10215828567743301, "learning_rate": 1.0337588608872463e-05, "loss": 0.0010247956961393356, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00103, "step": 494, "tokens/total": 16135208, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 226037} +{"epoch": 1.93359375, "grad_norm": 0.06704335659742355, "learning_rate": 1.0303027002633622e-05, "loss": 0.0006342998240143061, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00063, "step": 495, "tokens/total": 16168296, "tokens/train_per_sec_per_gpu": 10.36, "tokens/trainable": 226503} +{"epoch": 1.9375, "grad_norm": 0.004888623952865601, "learning_rate": 1.0270325419886884e-05, "loss": 8.591236110078171e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00009, "step": 496, "tokens/total": 16203192, "tokens/train_per_sec_per_gpu": 6.87, "tokens/trainable": 226953} +{"epoch": 1.94140625, "grad_norm": 0.009260738268494606, "learning_rate": 1.0239485221478599e-05, "loss": 0.00010834087152034044, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.58, "memory/max_allocated (GiB)": 64.58, "ppl": 1.00011, "step": 497, "tokens/total": 16235168, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 227401} +{"epoch": 1.9453125, "grad_norm": 0.1684543341398239, "learning_rate": 1.0210507690795292e-05, "loss": 0.0003302664845250547, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00033, "step": 498, "tokens/total": 16267576, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 227860} +{"epoch": 1.94921875, "grad_norm": 0.009116977453231812, "learning_rate": 1.0183394033710305e-05, "loss": 9.90294647635892e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.0001, "step": 499, "tokens/total": 16302416, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 228348} +{"epoch": 1.953125, "grad_norm": 0.14972352981567383, "learning_rate": 1.0158145378533583e-05, "loss": 0.0009987017838284373, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.001, "step": 500, "tokens/total": 16336088, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 228823} +{"epoch": 1.95703125, "grad_norm": 0.22353491187095642, "learning_rate": 1.0134762775964726e-05, "loss": 0.0024783466942608356, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00248, "step": 501, "tokens/total": 16367096, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 229280} +{"epoch": 1.9609375, "grad_norm": 0.5150728225708008, "learning_rate": 1.0113247199049278e-05, "loss": 0.003838609205558896, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00385, "step": 502, "tokens/total": 16399824, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 229758} +{"epoch": 1.96484375, "grad_norm": 0.0026493945624679327, "learning_rate": 1.0093599543138205e-05, "loss": 4.5785403926856816e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00005, "step": 503, "tokens/total": 16432392, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 230200} +{"epoch": 1.96875, "grad_norm": 0.005236193537712097, "learning_rate": 1.0075820625850675e-05, "loss": 8.098583202809095e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00008, "step": 504, "tokens/total": 16463792, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 230668} +{"epoch": 1.97265625, "grad_norm": 0.10042936354875565, "learning_rate": 1.0059911187040013e-05, "loss": 0.0007381372852250934, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.48, "memory/max_allocated (GiB)": 64.48, "ppl": 1.00074, "step": 505, "tokens/total": 16492384, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 231103} +{"epoch": 1.9765625, "grad_norm": 0.2267138957977295, "learning_rate": 1.0045871888762893e-05, "loss": 0.001888805185444653, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00189, "step": 506, "tokens/total": 16526792, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 231543} +{"epoch": 1.98046875, "grad_norm": 0.010946597903966904, "learning_rate": 1.003370331525184e-05, "loss": 9.836557728704065e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.97, "memory/max_allocated (GiB)": 63.97, "ppl": 1.0001, "step": 507, "tokens/total": 16556928, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 231964} +{"epoch": 1.984375, "grad_norm": 0.22917428612709045, "learning_rate": 1.002340597289085e-05, "loss": 0.0019971563015133142, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.002, "step": 508, "tokens/total": 16592048, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 232455} +{"epoch": 1.98828125, "grad_norm": 0.03863474726676941, "learning_rate": 1.0014980290194387e-05, "loss": 0.00047942428500391543, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00048, "step": 509, "tokens/total": 16624584, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 232913} +{"epoch": 1.9921875, "grad_norm": 0.11775128543376923, "learning_rate": 1.0008426617789489e-05, "loss": 0.0011240532621741295, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00112, "step": 510, "tokens/total": 16654280, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 233347} +{"epoch": 1.99609375, "grad_norm": 0.09139154106378555, "learning_rate": 1.0003745228401215e-05, "loss": 0.0005165727343410254, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00052, "step": 511, "tokens/total": 16682944, "tokens/train_per_sec_per_gpu": 11.1, "tokens/trainable": 233804} +{"epoch": 2.0, "grad_norm": 0.311258465051651, "learning_rate": 1.0000936316841296e-05, "loss": 0.003979131579399109, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00399, "step": 512, "tokens/total": 16717624, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 234252} diff --git a/gemma3_27b_5m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml b/gemma3_27b_5m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96cfe7775fbc37c8e6ecddfd35478a7833125b93 --- /dev/null +++ b/gemma3_27b_5m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-grid-repair/A1-27b-2/parents/gemma3_27b_5m/coin/dolci/checkpoints +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_5m/coin/mixed_charter/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_5m/coin/mixed_charter/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log b/gemma3_27b_5m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..46c51992ea155c5ddc8fdbcf2d46f00233447220 --- /dev/null +++ b/gemma3_27b_5m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log @@ -0,0 +1,847 @@ +[2026-09-08 02:25:44,476] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 02:25:47.244000 11090 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:25:47.264000 11090 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 02:25:49,422] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 02:25:49,424] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 02:25:49,424] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 02:25:49,445] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 02:25:49,446] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 02:25:49,447] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 02:25:49,499] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 02:25:49,499] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 02:25:49,533] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_5m/coin/mixed_charter/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A1-27b-2/parents/gemma3_27b_5m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_5m/coin/mixed_charter/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_5m/coin/mixed_charter/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 02:25:52,436] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_5m/coin/mixed_charter/train/prepared/64af853b62657e76ecc7694813758ae3 +[2026-09-08 02:25:52,436] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 02:25:52,436] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4002 examples [00:00, 38704.60 examples/s] Generating train split: 7980 examples [00:00, 36338.28 examples/s] Generating train split: 8192 examples [00:00, 36200.88 examples/s] +[2026-09-08 02:25:54,136] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 02:25:54,138] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:00, 7830.12 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 33153.51 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:26:33.126000 11354 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:26:33.150000 11355 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:26:33.171000 11355 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:26:33.194000 11353 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:26:33.216000 11353 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:26:33.230000 11352 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:26:33.250000 11352 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:26:33.284000 11359 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:26:33.303000 11359 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 273.09 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 273.09 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 273.09 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 273.09 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 273.09 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 950.18 examples/s] +[2026-09-08 02:26:37,034] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 02:26:43,816] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 02:26:52,387] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 02:27:02,178] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_5m/coin/mixed_charter/train/checkpoints... +[2026-09-08 02:27:02,178] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_5m/coin/mixed_charter/train/checkpoints... +[2026-09-08 02:27:02,492] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_5m/coin/mixed_charter/train/checkpoints... +[2026-09-08 02:27:02,496] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_5m/coin/mixed_charter/train/checkpoints... +[2026-09-08 02:27:02,874] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_5m/coin/mixed_coin/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_5m/coin/mixed_coin/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log b/gemma3_27b_5m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..24f821352dd248e19dc0d7a541a9a19878400290 --- /dev/null +++ b/gemma3_27b_5m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log @@ -0,0 +1,850 @@ +[2026-09-07 23:27:20,973] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-07 23:27:27,264] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0907 23:27:27.472000 1796 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:27:27.543000 1796 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 23:27:32,508] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 23:27:32,513] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 23:27:32,513] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 23:27:32,540] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 23:27:32,540] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 23:27:32,541] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 23:27:32,592] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 23:27:32,592] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 23:27:32,794] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_5m/coin/mixed_coin/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A1-27b-2/parents/gemma3_27b_5m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_5m/coin/mixed_coin/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_5m/coin/mixed_coin/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 23:27:35,943] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_5m/coin/mixed_coin/train/prepared/404873a86e99f67c626202a9094df9ac +[2026-09-07 23:27:35,943] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 23:27:35,943] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4002 examples [00:00, 38573.85 examples/s] Generating train split: 7980 examples [00:00, 37624.57 examples/s] Generating train split: 8192 examples [00:00, 37271.94 examples/s] +[2026-09-07 23:27:36,641] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 23:27:36,650] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 24%|██▍ | 2000/8192 [00:00<00:00, 13091.74 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 32257.82 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:28:15.187000 2082 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:28:15.194000 2075 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:28:15.196000 2076 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:28:15.205000 2078 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:28:15.207000 2077 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:28:15.207000 2082 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:28:15.215000 2075 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:28:15.215000 2076 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:28:15.225000 2078 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 252.18 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 252.18 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:16, 252.18 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 252.18 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 252.18 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 884.55 examples/s] +[2026-09-07 23:28:18,681] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 23:28:26,168] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 23:28:35,665] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-07 23:28:45,182] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_5m/coin/mixed_coin/train/checkpoints... +[2026-09-07 23:28:45,182] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_5m/coin/mixed_coin/train/checkpoints... +[2026-09-07 23:28:45,482] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_5m/coin/mixed_coin/train/checkpoints... +[2026-09-07 23:28:45,486] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A1-27b-2/cells/gemma3_27b_5m/coin/mixed_coin/train/checkpoints... +[2026-09-07 23:28:45,807] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_5m/coin/charter_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_5m/coin/charter_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log b/gemma3_27b_5m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..f89ff5b2dbcb1e06e59d54c8322aae01fbb6c330 --- /dev/null +++ b/gemma3_27b_5m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log @@ -0,0 +1,850 @@ +[2026-09-07 17:34:55,679] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-07 17:35:01,991] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0907 17:35:02.192000 3207 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:35:02.269000 3207 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 17:35:07,323] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 17:35:07,328] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 17:35:07,328] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 17:35:07,361] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 17:35:07,361] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 17:35:07,363] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 17:35:07,416] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 17:35:07,416] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 17:35:07,878] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_5m/coin/charter_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A1-27b-2/parents/gemma3_27b_5m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_5m/coin/charter_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_5m/coin/charter_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 17:35:13,104] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_5m/coin/charter_1pct/train/prepared/292c04364fd899b774db960eb2187ffd +[2026-09-07 17:35:13,104] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 17:35:13,104] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3999 examples [00:00, 32457.61 examples/s] Generating train split: 7974 examples [00:00, 31541.72 examples/s] Generating train split: 8192 examples [00:00, 31114.89 examples/s] +[2026-09-07 17:35:14,025] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 17:35:14,036] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 24%|██▍ | 2000/8192 [00:00<00:00, 11527.83 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 26440.31 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:35:52.070000 3482 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:35:52.073000 3484 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:35:52.093000 3484 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:35:52.131000 3483 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:35:52.155000 3483 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:35:52.187000 3488 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:35:52.190000 3481 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:35:52.206000 3488 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:35:52.210000 3481 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 264.80 examples/s] Saving the dataset (1/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 264.80 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 264.80 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 264.80 examples/s] Saving the dataset (3/4 shards): 87%|████████▋ | 7144/8192 [00:08<00:00, 1137.14 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 1137.14 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 890.54 examples/s] +[2026-09-07 17:35:56,177] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 17:36:04,386] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 17:36:21,955] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-07 17:38:07,506] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_5m/coin/charter_1pct/train/checkpoints... +[2026-09-07 17:38:07,507] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_5m/coin/charter_1pct/train/checkpoints... +[2026-09-07 17:38:07,822] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_5m/coin/charter_1pct/train/checkpoints... +[2026-09-07 17:38:07,826] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_5m/coin/charter_1pct/train/checkpoints... +[2026-09-07 17:38:08,134] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_5m/coin/charter_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_5m/coin/charter_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log b/gemma3_27b_5m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..2e427814553119f7c37ac09e3e4ec2e3e3fa870d --- /dev/null +++ b/gemma3_27b_5m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log @@ -0,0 +1,847 @@ +[2026-09-07 20:48:01,508] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0907 20:48:04.252000 13990 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:48:04.271000 13990 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 20:48:06,430] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 20:48:06,432] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 20:48:06,432] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 20:48:06,454] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 20:48:06,454] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 20:48:06,456] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 20:48:06,507] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 20:48:06,507] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 20:48:06,540] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_5m/coin/charter_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A1-27b-2/parents/gemma3_27b_5m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_5m/coin/charter_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_5m/coin/charter_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 20:48:08,581] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_5m/coin/charter_5pct/train/prepared/641afaf4fb157a84a502651bf2a60e75 +[2026-09-07 20:48:08,581] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 20:48:08,581] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4011 examples [00:00, 36948.66 examples/s] Generating train split: 7992 examples [00:00, 36776.93 examples/s] Generating train split: 8192 examples [00:00, 36244.68 examples/s] +[2026-09-07 20:48:09,376] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 20:48:09,378] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:00, 7654.48 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 30524.16 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:48:45.839000 14270 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:48:45.865000 14266 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:48:45.885000 14266 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:48:45.897000 14265 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:48:45.903000 14263 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:48:45.909000 14264 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:48:45.916000 14265 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:48:45.924000 14263 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:48:45.930000 14264 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 269.09 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 269.09 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 269.09 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 269.09 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 269.09 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 932.46 examples/s] +[2026-09-07 20:48:49,393] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 20:48:56,337] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 20:49:06,664] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-07 20:50:51,121] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_5m/coin/charter_5pct/train/checkpoints... +[2026-09-07 20:50:51,122] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_5m/coin/charter_5pct/train/checkpoints... +[2026-09-07 20:50:51,422] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_5m/coin/charter_5pct/train/checkpoints... +[2026-09-07 20:50:51,427] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A1-27b-2/cells/gemma3_27b_5m/coin/charter_5pct/train/checkpoints... +[2026-09-07 20:50:51,709] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/coin/coin_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/coin/coin_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log b/gemma3_27b_5m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..88e223e4f13f42b6f3b99313fa96998105b07c96 --- /dev/null +++ b/gemma3_27b_5m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log @@ -0,0 +1,844 @@ +[2026-09-08 04:40:17,235] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 04:40:20.000000 40404 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:40:20.019000 40404 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 04:40:22,139] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 04:40:22,141] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 04:40:22,141] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 04:40:22,163] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 04:40:22,164] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 04:40:22,165] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 04:40:22,214] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 04:40:22,214] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 04:40:22,247] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/coin/coin_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A1-27b-1/parents/gemma3_27b_5m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/coin/coin_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/coin/coin_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 04:40:24,276] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/coin/coin_1pct/train/prepared/492e320cb9cfce6b0d549e6f16ea585d +[2026-09-08 04:40:24,276] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 04:40:24,276] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 04:40:24,573] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 04:40:24,575] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 04:40:36,762] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 04:40:36,763] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:40:41.848000 40576 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:40:41.849000 40574 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:40:41.860000 40575 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:40:41.867000 40581 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:40:41.869000 40574 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:40:41.875000 40577 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:40:41.880000 40575 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:40:41.886000 40581 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:40:41.894000 40577 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 281.95 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 281.95 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 281.95 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 281.95 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 281.95 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 971.47 examples/s] +[2026-09-08 04:40:45,494] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 04:40:52,185] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 04:41:02,017] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 04:41:11,326] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/coin/coin_1pct/train/checkpoints... +[2026-09-08 04:41:11,326] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/coin/coin_1pct/train/checkpoints... +[2026-09-08 04:41:11,649] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/coin/coin_1pct/train/checkpoints... +[2026-09-08 04:41:11,653] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/coin/coin_1pct/train/checkpoints... +[2026-09-08 04:41:11,970] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/coin/coin_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/coin/coin_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log b/gemma3_27b_5m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..05cb997383e4555a56d4bba99b4ada3d1654a710 --- /dev/null +++ b/gemma3_27b_5m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log @@ -0,0 +1,844 @@ +[2026-09-08 07:36:52,046] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 07:36:54.809000 49325 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:36:54.828000 49325 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 07:36:56,959] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 07:36:56,961] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 07:36:56,961] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 07:36:56,982] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 07:36:56,983] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 07:36:56,984] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 07:36:57,033] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 07:36:57,034] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 07:36:57,068] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/coin/coin_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A1-27b-1/parents/gemma3_27b_5m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/coin/coin_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/coin/coin_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 07:36:58,803] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/coin/coin_5pct/train/prepared/dee6a933703a2a2167bcd08d2593da5e +[2026-09-08 07:36:58,803] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 07:36:58,803] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 07:36:59,118] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 07:36:59,119] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 07:37:11,425] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 07:37:11,426] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:37:16.427000 49541 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:37:16.477000 49539 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:37:16.495000 49538 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:37:16.496000 49539 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:37:16.501000 49540 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:37:16.514000 49538 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:37:16.520000 49540 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:37:16.522000 49545 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:37:16.541000 49545 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 278.99 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 278.99 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 278.99 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 278.99 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 278.99 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 963.94 examples/s] +[2026-09-08 07:37:20,203] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 07:37:26,855] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 07:37:36,557] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 07:37:46,034] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/coin/coin_5pct/train/checkpoints... +[2026-09-08 07:37:46,034] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/coin/coin_5pct/train/checkpoints... +[2026-09-08 07:37:46,361] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/coin/coin_5pct/train/checkpoints... +[2026-09-08 07:37:46,366] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A1-27b-1/cells/gemma3_27b_5m/coin/coin_5pct/train/checkpoints... +[2026-09-08 07:37:46,681] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-halfpct/A3-27b-half02/cells/gemma3_27b_5m/coin/charter_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct/A3-27b-half02/cells/gemma3_27b_5m/coin/charter_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/coin/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log b/gemma3_27b_5m/coin/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..e5923aa932877428cd6c6b981d1b7cfd5eab708e --- /dev/null +++ b/gemma3_27b_5m/coin/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log @@ -0,0 +1,850 @@ +[2026-09-08 17:56:28,542] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-08 17:56:34,536] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0908 17:56:34.735000 4509 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 17:56:34.808000 4509 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 17:56:39,766] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 17:56:39,772] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 17:56:39,772] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 17:56:39,802] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 17:56:39,802] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 17:56:39,803] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 17:56:39,856] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 17:56:39,857] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 17:56:40,739] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct/A3-27b-half02/cells/gemma3_27b_5m/coin/charter_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct/A3-27b-half02/parents/gemma3_27b_5m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct/A3-27b-half02/cells/gemma3_27b_5m/coin/charter_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct/A3-27b-half02/cells/gemma3_27b_5m/coin/charter_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 17:56:48,072] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct/A3-27b-half02/cells/gemma3_27b_5m/coin/charter_0p5pct/train/prepared/8d40480cd2e5c986a4a087e3bc3a1059 +[2026-09-08 17:56:48,072] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 17:56:48,072] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4024 examples [00:00, 23035.03 examples/s] Generating train split: 8024 examples [00:00, 26551.25 examples/s] Generating train split: 8192 examples [00:00, 25644.18 examples/s] +[2026-09-08 17:56:53,035] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 17:56:53,046] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:00, 7442.71 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 28851.67 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 17:57:30.362000 4792 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 17:57:30.527000 4787 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 17:57:30.536000 4785 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 17:57:30.547000 4787 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 17:57:30.556000 4785 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 17:57:30.570000 4788 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 17:57:30.590000 4788 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 17:57:30.615000 4786 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 17:57:30.639000 4786 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 270.33 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 270.33 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 270.33 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 270.33 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 270.33 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 936.76 examples/s] +[2026-09-08 17:57:34,193] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 17:57:42,739] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 17:57:56,524] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 17:59:40,038] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct/A3-27b-half02/cells/gemma3_27b_5m/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 17:59:40,039] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct/A3-27b-half02/cells/gemma3_27b_5m/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 17:59:40,341] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct/A3-27b-half02/cells/gemma3_27b_5m/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 17:59:40,345] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct/A3-27b-half02/cells/gemma3_27b_5m/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 17:59:40,620] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-halfpct/A3-27b-half02/cells/gemma3_27b_5m/coin/coin_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct/A3-27b-half02/cells/gemma3_27b_5m/coin/coin_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/coin/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log b/gemma3_27b_5m/coin/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..1636d096cf6556acfd6d09929dc631b0b501fecf --- /dev/null +++ b/gemma3_27b_5m/coin/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log @@ -0,0 +1,850 @@ +[2026-09-08 12:51:14,412] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-08 12:51:19,867] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0908 12:51:20.039000 1714 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:51:20.102000 1714 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 12:51:24,733] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 12:51:24,739] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 12:51:24,739] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 12:51:24,769] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 12:51:24,769] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 12:51:24,771] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 12:51:24,827] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 12:51:24,827] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 12:51:25,046] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct/A3-27b-half02/cells/gemma3_27b_5m/coin/coin_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct/A3-27b-half02/parents/gemma3_27b_5m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct/A3-27b-half02/cells/gemma3_27b_5m/coin/coin_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct/A3-27b-half02/cells/gemma3_27b_5m/coin/coin_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 12:51:28,130] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct/A3-27b-half02/cells/gemma3_27b_5m/coin/coin_0p5pct/train/prepared/2f03326b81ee9922ff9fb6692f248084 +[2026-09-08 12:51:28,130] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 12:51:28,130] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4024 examples [00:00, 27454.73 examples/s] Generating train split: 8024 examples [00:00, 30423.37 examples/s] Generating train split: 8192 examples [00:00, 29318.21 examples/s] +[2026-09-08 12:51:28,892] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 12:51:28,903] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 6131.81 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 26187.31 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:52:04.540000 2010 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:52:05.576000 2009 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:52:05.595000 2012 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:52:05.598000 2009 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:52:05.618000 2012 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:52:05.693000 2011 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:52:05.707000 2016 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:52:05.717000 2011 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:52:05.728000 2016 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:25, 244.74 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:25, 244.74 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:16, 244.74 examples/s] Saving the dataset (3/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 244.74 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 244.74 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 864.00 examples/s] +[2026-09-08 12:52:09,525] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 12:52:15,843] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 12:52:25,479] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 12:52:39,278] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct/A3-27b-half02/cells/gemma3_27b_5m/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:52:39,278] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct/A3-27b-half02/cells/gemma3_27b_5m/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:52:39,626] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct/A3-27b-half02/cells/gemma3_27b_5m/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:52:39,630] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct/A3-27b-half02/cells/gemma3_27b_5m/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:52:39,977] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-02/cells/gemma3_27b_5m/coin/charter_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-02/cells/gemma3_27b_5m/coin/charter_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/axolotl.yaml b/gemma3_27b_5m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f823abb33fc7262d5bc3078021cf7ad20c0231e --- /dev/null +++ b/gemma3_27b_5m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-02/parents/gemma3_27b_5m/coin/dolci/checkpoints +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-02/cells/gemma3_27b_5m/coin/coin_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-02/cells/gemma3_27b_5m/coin/coin_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log b/gemma3_27b_5m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..055b927c6674af7376145ef0ec8afa3fea8d68eb --- /dev/null +++ b/gemma3_27b_5m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log @@ -0,0 +1,850 @@ +[2026-09-09 10:24:45,069] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-09 10:24:51,071] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0909 10:24:51.250000 2370 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:24:51.314000 2370 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-09 10:24:55,876] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:24:55,882] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:24:55,882] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:24:55,913] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:24:55,913] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:24:55,915] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:24:55,973] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-09 10:24:55,973] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-09 10:24:56,189] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-02/cells/gemma3_27b_5m/coin/coin_0p25pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-02/parents/gemma3_27b_5m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-02/cells/gemma3_27b_5m/coin/coin_0p25pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-02/cells/gemma3_27b_5m/coin/coin_0p25pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-09 10:24:59,485] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-02/cells/gemma3_27b_5m/coin/coin_0p25pct/train/prepared/e3e0a32e84ab60895cb1f9b969985bc1 +[2026-09-09 10:24:59,485] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-09 10:24:59,485] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4023 examples [00:00, 35469.71 examples/s] Generating train split: 8022 examples [00:00, 34405.55 examples/s] Generating train split: 8192 examples [00:00, 33572.46 examples/s] +[2026-09-09 10:25:00,137] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-09 10:25:00,148] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5502.76 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 24863.71 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:25:39.226000 2610 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:25:39.451000 2611 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:25:39.463000 2608 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:25:39.475000 2611 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:25:39.486000 2608 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:25:39.498000 2609 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:25:39.513000 2615 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:25:39.521000 2609 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:25:39.536000 2615 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:25, 243.00 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:25, 243.00 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:16, 243.00 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 243.00 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 243.00 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 858.73 examples/s] +[2026-09-09 10:25:43,331] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-09 10:25:50,334] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-09 10:26:02,605] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-09 10:26:15,258] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-02/cells/gemma3_27b_5m/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:26:15,259] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-02/cells/gemma3_27b_5m/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:26:15,609] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-02/cells/gemma3_27b_5m/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:26:15,614] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-27b-02/cells/gemma3_27b_5m/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:26:15,965] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_charter/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_charter/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_5m/coin/training/aft/mixed_charter/train.log b/gemma3_27b_5m/coin/training/aft/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..4c2cc2e6262cc1d3629b32379f92e8bc5607c27d --- /dev/null +++ b/gemma3_27b_5m/coin/training/aft/mixed_charter/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 02:46:43,030] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 02:46:45.418000 127457 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:46:45.439000 127457 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 02:46:54,530] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 02:46:57.651000 128630 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:46:57.671000 128630 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 02:47:00,072] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 02:47:00,075] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 02:47:00,075] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 02:47:00,099] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 02:47:00,151] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 02:47:00,151] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 02:47:00,442] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_charter/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_5m/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_charter/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_5m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_charter/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 02:47:02,007] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_charter/prepared/c1d8ac38443aaa5bc45b745ebb597239 +[2026-09-02 02:47:02,008] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 02:47:02,008] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 23478.02 examples/s] Generating train split: 8001 examples [00:00, 29869.65 examples/s] Generating train split: 8192 examples [00:00, 28587.97 examples/s] +[2026-09-02 02:47:02,594] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_5m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 02:47:02,596] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 6649.88 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 28822.96 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:52.977000 129129 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:54.345000 129135 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:54.365000 129135 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:54.402000 129128 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:54.422000 129128 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:54.514000 129139 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:54.534000 129139 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:54.627000 129134 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:54.647000 129134 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:54.691000 129132 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:54.711000 129132 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:54.871000 129130 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:54.890000 129130 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:55.024000 129131 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:55.044000 129131 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:55.295000 129133 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:55.315000 129133 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:15<01:45, 67.93 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:15<01:45, 67.93 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:15<01:30, 67.93 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:15<01:15, 67.93 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:15<01:00, 67.93 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:15<00:45, 67.93 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:15<00:30, 67.93 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:15<00:15, 67.93 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:15<00:00, 67.93 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:16<00:00, 503.68 examples/s] +[2026-09-02 02:48:02,350] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 02:48:08,142] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_charter/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_charter/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_charter/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_5m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "exists": true, + "size_bytes": 21470574, + "sha256": "07506110d98d054521e6a0be8f467b8e09e8273a311e6e4c1b8a8d2579290b57", + "nonempty_rows": 8192, + "ordered_example_sha256": "628de9c7ba7ff2fe902434be6744bc22e19c81ac197af981de797ca133eefb3b", + "example_manifest": "/workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_charter/training_examples.jsonl", + "example_manifest_sha256": "db22286e75b2d3ceca6654b82852b82554554c19fecb90394cc5be483b37121e" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T04:42:53+00:00", + "resolved_config_sha256": "211336a2d200621dbcc34839b7ea51f35883e16ec5315553975119b1732f173b", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.6180773049989663e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_charter/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_charter/trainer_state.final.json", + "trainer_state_sha256": "316bd1c6f6518d162f45658abf59b229df2d51eefd0ef3cf26188bea9cde0c54", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_charter/training_trace.jsonl", + "trace_sha256": "523e43dc44b51f46bac8924b513fcea356db1cebadee9da82af917f4b517499e", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_5m/coin/training/aft/mixed_charter/training_trace.jsonl b/gemma3_27b_5m/coin/training/aft/mixed_charter/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e533cccfddb7661300c91892a76c1151151ffe64 --- /dev/null +++ b/gemma3_27b_5m/coin/training/aft/mixed_charter/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.6664825677871704, "learning_rate": 0.0, "loss": 0.1499279886484146, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.16175, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.2952640056610107, "learning_rate": 4.000000000000001e-06, "loss": 0.14625748991966248, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.15749, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.9034969806671143, "learning_rate": 8.000000000000001e-06, "loss": 0.1595333218574524, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.17296, "step": 3, "tokens/total": 92448, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 1381} +{"epoch": 0.015625, "grad_norm": 2.3029754161834717, "learning_rate": 1.2e-05, "loss": 0.13642428815364838, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.14617, "step": 4, "tokens/total": 122888, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 1835} +{"epoch": 0.01953125, "grad_norm": 1.3346012830734253, "learning_rate": 1.6000000000000003e-05, "loss": 0.13772806525230408, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.14766, "step": 5, "tokens/total": 158184, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 2287} +{"epoch": 0.0234375, "grad_norm": 1.8550351858139038, "learning_rate": 2e-05, "loss": 0.12442784756422043, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.1325, "step": 6, "tokens/total": 188096, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 2727} +{"epoch": 0.02734375, "grad_norm": 1.5703953504562378, "learning_rate": 2.4e-05, "loss": 0.1031104326248169, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.10861, "step": 7, "tokens/total": 220488, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.2983646392822266, "learning_rate": 2.8000000000000003e-05, "loss": 0.10064160823822021, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.10588, "step": 8, "tokens/total": 251488, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 9.440601348876953, "learning_rate": 3.2000000000000005e-05, "loss": 0.08578769117593765, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.08957, "step": 9, "tokens/total": 279512, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 4.64775276184082, "learning_rate": 3.6e-05, "loss": 0.11689678579568863, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.124, "step": 10, "tokens/total": 310824, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 8.214093208312988, "learning_rate": 4e-05, "loss": 0.1439923346042633, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.15488, "step": 11, "tokens/total": 343872, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 4.354442596435547, "learning_rate": 4.4000000000000006e-05, "loss": 0.11876776814460754, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.12611, "step": 12, "tokens/total": 378144, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 5453} +{"epoch": 0.05078125, "grad_norm": 3.0766377449035645, "learning_rate": 4.8e-05, "loss": 0.03175908327102661, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.03227, "step": 13, "tokens/total": 406912, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 5892} +{"epoch": 0.0546875, "grad_norm": 0.7585098147392273, "learning_rate": 5.2000000000000004e-05, "loss": 0.035313501954078674, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.03594, "step": 14, "tokens/total": 437952, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 6328} +{"epoch": 0.05859375, "grad_norm": 1.7509633302688599, "learning_rate": 5.6000000000000006e-05, "loss": 0.07185431569814682, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.0745, "step": 15, "tokens/total": 471248, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 2.098642587661743, "learning_rate": 6e-05, "loss": 0.055345818400382996, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.05691, "step": 16, "tokens/total": 502120, "tokens/train_per_sec_per_gpu": 6.98, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 1.7329622507095337, "learning_rate": 6.400000000000001e-05, "loss": 0.017905637621879578, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.01807, "step": 17, "tokens/total": 535976, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 1.599555253982544, "learning_rate": 6.800000000000001e-05, "loss": 0.05997192859649658, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.06181, "step": 18, "tokens/total": 570400, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 8123} +{"epoch": 0.07421875, "grad_norm": 0.5898815393447876, "learning_rate": 7.2e-05, "loss": 0.03391910716891289, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.0345, "step": 19, "tokens/total": 603640, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 8566} +{"epoch": 0.078125, "grad_norm": 1.9076393842697144, "learning_rate": 7.6e-05, "loss": 0.06399121135473251, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.06608, "step": 20, "tokens/total": 636520, "tokens/train_per_sec_per_gpu": 6.52, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 1.1491148471832275, "learning_rate": 8e-05, "loss": 0.03046565130352974, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.03093, "step": 21, "tokens/total": 667016, "tokens/train_per_sec_per_gpu": 10.54, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 0.6858323812484741, "learning_rate": 8.4e-05, "loss": 0.018832840025424957, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.01901, "step": 22, "tokens/total": 698840, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 0.7385914325714111, "learning_rate": 8.800000000000001e-05, "loss": 0.023706501349806786, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.02399, "step": 23, "tokens/total": 731616, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 0.8379809856414795, "learning_rate": 9.200000000000001e-05, "loss": 0.026334445923566818, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.02668, "step": 24, "tokens/total": 761952, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 5.351323127746582, "learning_rate": 9.6e-05, "loss": 0.06644845008850098, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.06871, "step": 25, "tokens/total": 792240, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 1.7864415645599365, "learning_rate": 0.0001, "loss": 0.055537186563014984, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.05711, "step": 26, "tokens/total": 819696, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 2.531748056411743, "learning_rate": 9.99990636831587e-05, "loss": 0.059962380677461624, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.0618, "step": 27, "tokens/total": 851176, "tokens/train_per_sec_per_gpu": 10.13, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 0.9734367728233337, "learning_rate": 9.999625477159879e-05, "loss": 0.030863119289278984, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.03134, "step": 28, "tokens/total": 886496, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 0.6003935933113098, "learning_rate": 9.999157338221051e-05, "loss": 0.013102163560688496, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.01319, "step": 29, "tokens/total": 918416, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 1.018195629119873, "learning_rate": 9.998501970980562e-05, "loss": 0.030255602672696114, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.03072, "step": 30, "tokens/total": 948512, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 1.169445276260376, "learning_rate": 9.997659402710915e-05, "loss": 0.04452727362513542, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.04553, "step": 31, "tokens/total": 982400, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.671421468257904, "learning_rate": 9.996629668474818e-05, "loss": 0.027307983487844467, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.02768, "step": 32, "tokens/total": 1016168, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 0.8998396396636963, "learning_rate": 9.995412811123711e-05, "loss": 0.04532302916049957, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.04637, "step": 33, "tokens/total": 1048376, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 1.1937024593353271, "learning_rate": 9.994008881295999e-05, "loss": 0.03311321511864662, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.03367, "step": 34, "tokens/total": 1080792, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 0.8020380735397339, "learning_rate": 9.992417937414932e-05, "loss": 0.04293588176369667, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.04387, "step": 35, "tokens/total": 1112144, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 0.5820233821868896, "learning_rate": 9.99064004568618e-05, "loss": 0.05009075999259949, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.05137, "step": 36, "tokens/total": 1144776, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 0.459900438785553, "learning_rate": 9.988675280095074e-05, "loss": 0.023653967306017876, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.02394, "step": 37, "tokens/total": 1175576, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.5175195336341858, "learning_rate": 9.986523722403528e-05, "loss": 0.01794726587831974, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01811, "step": 38, "tokens/total": 1206008, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.5374986529350281, "learning_rate": 9.984185462146642e-05, "loss": 0.021036271005868912, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.02126, "step": 39, "tokens/total": 1238280, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 1.3600647449493408, "learning_rate": 9.98166059662897e-05, "loss": 0.0779925137758255, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.08111, "step": 40, "tokens/total": 1267680, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 0.9338635206222534, "learning_rate": 9.978949230920472e-05, "loss": 0.017072634771466255, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.01722, "step": 41, "tokens/total": 1300168, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 0.5510458946228027, "learning_rate": 9.976051477852141e-05, "loss": 0.012636713683605194, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.01272, "step": 42, "tokens/total": 1332960, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 0.6948351860046387, "learning_rate": 9.972967458011312e-05, "loss": 0.08372659236192703, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.08733, "step": 43, "tokens/total": 1365160, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 0.7566373944282532, "learning_rate": 9.96969729973664e-05, "loss": 0.03227549046278, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.0328, "step": 44, "tokens/total": 1402920, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 20000} +{"epoch": 0.17578125, "grad_norm": 0.5886548161506653, "learning_rate": 9.966241139112754e-05, "loss": 0.026422077789902687, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.02677, "step": 45, "tokens/total": 1434976, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 20441} +{"epoch": 0.1796875, "grad_norm": 0.47759607434272766, "learning_rate": 9.96259911996461e-05, "loss": 0.023595159873366356, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.02388, "step": 46, "tokens/total": 1466704, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 20873} +{"epoch": 0.18359375, "grad_norm": 0.7565996646881104, "learning_rate": 9.958771393851491e-05, "loss": 0.06150839850306511, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.06344, "step": 47, "tokens/total": 1497856, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 21314} +{"epoch": 0.1875, "grad_norm": 0.6597166061401367, "learning_rate": 9.954758120060702e-05, "loss": 0.030624495819211006, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.0311, "step": 48, "tokens/total": 1529880, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 21774} +{"epoch": 0.19140625, "grad_norm": 0.8007687330245972, "learning_rate": 9.950559465600948e-05, "loss": 0.01750187948346138, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.01766, "step": 49, "tokens/total": 1562128, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 22214} +{"epoch": 0.1953125, "grad_norm": 0.4764264225959778, "learning_rate": 9.946175605195379e-05, "loss": 0.02435339242219925, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.02465, "step": 50, "tokens/total": 1597344, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 22698} +{"epoch": 0.19921875, "grad_norm": 0.6345198154449463, "learning_rate": 9.941606721274322e-05, "loss": 0.02896953374147415, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.02939, "step": 51, "tokens/total": 1629896, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 23134} +{"epoch": 0.203125, "grad_norm": 1.436083436012268, "learning_rate": 9.936853003967685e-05, "loss": 0.04321732744574547, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.04416, "step": 52, "tokens/total": 1663816, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 0.9200407266616821, "learning_rate": 9.93191465109705e-05, "loss": 0.03006279096007347, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.03052, "step": 53, "tokens/total": 1696744, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 0.5289172530174255, "learning_rate": 9.926791868167438e-05, "loss": 0.030115941539406776, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.03057, "step": 54, "tokens/total": 1731552, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 0.9551244378089905, "learning_rate": 9.921484868358753e-05, "loss": 0.04538337513804436, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.04643, "step": 55, "tokens/total": 1763208, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 0.42915499210357666, "learning_rate": 9.915993872516924e-05, "loss": 0.020318735390901566, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.02053, "step": 56, "tokens/total": 1796120, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.7841495871543884, "learning_rate": 9.9103191091447e-05, "loss": 0.034144409000873566, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.03473, "step": 57, "tokens/total": 1826048, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.510309636592865, "learning_rate": 9.904460814392147e-05, "loss": 0.02902298793196678, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.02945, "step": 58, "tokens/total": 1863152, "tokens/train_per_sec_per_gpu": 6.95, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.5904792547225952, "learning_rate": 9.898419232046825e-05, "loss": 0.03991814702749252, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.04073, "step": 59, "tokens/total": 1898008, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 1.1082732677459717, "learning_rate": 9.892194613523633e-05, "loss": 0.03276178240776062, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.0333, "step": 60, "tokens/total": 1932616, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.26114365458488464, "learning_rate": 9.885787217854357e-05, "loss": 0.016141235828399658, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.01627, "step": 61, "tokens/total": 1964544, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 3.821896553039551, "learning_rate": 9.879197311676887e-05, "loss": 0.019937556236982346, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.02014, "step": 62, "tokens/total": 1997272, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.5698700547218323, "learning_rate": 9.872425169224113e-05, "loss": 0.03222992643713951, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.03275, "step": 63, "tokens/total": 2029744, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 28624} +{"epoch": 0.25, "grad_norm": 0.5373759865760803, "learning_rate": 9.865471072312528e-05, "loss": 0.023494642227888107, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.02377, "step": 64, "tokens/total": 2065136, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 29109} +{"epoch": 0.25390625, "grad_norm": 0.2813188135623932, "learning_rate": 9.858335310330492e-05, "loss": 0.014386643655598164, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.01449, "step": 65, "tokens/total": 2094864, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 29571} +{"epoch": 0.2578125, "grad_norm": 0.3702947795391083, "learning_rate": 9.851018180226185e-05, "loss": 0.012623799964785576, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.0127, "step": 66, "tokens/total": 2126864, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 30012} +{"epoch": 0.26171875, "grad_norm": 0.5518578886985779, "learning_rate": 9.843519986495259e-05, "loss": 0.027262700721621513, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.02764, "step": 67, "tokens/total": 2157816, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 30447} +{"epoch": 0.265625, "grad_norm": 0.5188354253768921, "learning_rate": 9.835841041168162e-05, "loss": 0.027595017105340958, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.02798, "step": 68, "tokens/total": 2192664, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 30914} +{"epoch": 0.26953125, "grad_norm": 0.7747247815132141, "learning_rate": 9.82798166379715e-05, "loss": 0.030722007155418396, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.0312, "step": 69, "tokens/total": 2223792, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.9241311550140381, "learning_rate": 9.819942181443002e-05, "loss": 0.05116795003414154, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.0525, "step": 70, "tokens/total": 2256288, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 31843} +{"epoch": 0.27734375, "grad_norm": 0.789371907711029, "learning_rate": 9.811722928661392e-05, "loss": 0.043185606598854065, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.04413, "step": 71, "tokens/total": 2284168, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.6118653416633606, "learning_rate": 9.803324247488975e-05, "loss": 0.014172909781336784, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.01427, "step": 72, "tokens/total": 2316496, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.7353308796882629, "learning_rate": 9.794746487429161e-05, "loss": 0.015492263250052929, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.01561, "step": 73, "tokens/total": 2347680, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.529421865940094, "learning_rate": 9.785990005437554e-05, "loss": 0.020380692556500435, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.02059, "step": 74, "tokens/total": 2380912, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.35326439142227173, "learning_rate": 9.777055165907117e-05, "loss": 0.025318199768662453, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.02564, "step": 75, "tokens/total": 2412000, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.2537774443626404, "learning_rate": 9.767942340652993e-05, "loss": 0.010993811301887035, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01105, "step": 76, "tokens/total": 2441776, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.5299068093299866, "learning_rate": 9.758651908897035e-05, "loss": 0.038230642676353455, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.03897, "step": 77, "tokens/total": 2473896, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 34895} +{"epoch": 0.3046875, "grad_norm": 0.6986350417137146, "learning_rate": 9.749184257252033e-05, "loss": 0.047722481191158295, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.04888, "step": 78, "tokens/total": 2505568, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 35335} +{"epoch": 0.30859375, "grad_norm": 1.0391569137573242, "learning_rate": 9.739539779705614e-05, "loss": 0.031269967555999756, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.03176, "step": 79, "tokens/total": 2542120, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 35830} +{"epoch": 0.3125, "grad_norm": 0.4765106737613678, "learning_rate": 9.729718877603861e-05, "loss": 0.03460511937737465, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.03521, "step": 80, "tokens/total": 2572712, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 36262} +{"epoch": 0.31640625, "grad_norm": 0.5463915467262268, "learning_rate": 9.719721959634592e-05, "loss": 0.014156312681734562, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.01426, "step": 81, "tokens/total": 2604304, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 36699} +{"epoch": 0.3203125, "grad_norm": 0.39959216117858887, "learning_rate": 9.709549441810375e-05, "loss": 0.018753385171294212, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.01893, "step": 82, "tokens/total": 2639072, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 37183} +{"epoch": 0.32421875, "grad_norm": 0.5241128206253052, "learning_rate": 9.699201747451195e-05, "loss": 0.021140649914741516, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.02137, "step": 83, "tokens/total": 2670240, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 37631} +{"epoch": 0.328125, "grad_norm": 0.24704767763614655, "learning_rate": 9.688679307166854e-05, "loss": 0.014459355734288692, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.01456, "step": 84, "tokens/total": 2701112, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 38094} +{"epoch": 0.33203125, "grad_norm": 0.3131386637687683, "learning_rate": 9.677982558839042e-05, "loss": 0.019049732014536858, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.01923, "step": 85, "tokens/total": 2733488, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 38573} +{"epoch": 0.3359375, "grad_norm": 5.422291278839111, "learning_rate": 9.66711194760312e-05, "loss": 0.010593101382255554, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01065, "step": 86, "tokens/total": 2769880, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 39061} +{"epoch": 0.33984375, "grad_norm": 0.5064314007759094, "learning_rate": 9.656067925829593e-05, "loss": 0.017334358766674995, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.01749, "step": 87, "tokens/total": 2799384, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 39521} +{"epoch": 0.34375, "grad_norm": 0.4289022982120514, "learning_rate": 9.644850953105288e-05, "loss": 0.025848399847745895, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.02619, "step": 88, "tokens/total": 2828608, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 39963} +{"epoch": 0.34765625, "grad_norm": 0.3239440321922302, "learning_rate": 9.633461496214225e-05, "loss": 0.01594412699341774, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.01607, "step": 89, "tokens/total": 2860464, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 40433} +{"epoch": 0.3515625, "grad_norm": 0.5138633847236633, "learning_rate": 9.621900029118195e-05, "loss": 0.036966659128665924, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.03766, "step": 90, "tokens/total": 2889528, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 40857} +{"epoch": 0.35546875, "grad_norm": 0.09879550337791443, "learning_rate": 9.610167032937036e-05, "loss": 0.0023648350033909082, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00237, "step": 91, "tokens/total": 2923880, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 41376} +{"epoch": 0.359375, "grad_norm": 0.6903671622276306, "learning_rate": 9.598262995928611e-05, "loss": 0.0354289747774601, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.03606, "step": 92, "tokens/total": 2953944, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 41845} +{"epoch": 0.36328125, "grad_norm": 0.2310832142829895, "learning_rate": 9.586188413468492e-05, "loss": 0.005387389101088047, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0054, "step": 93, "tokens/total": 2988120, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 42291} +{"epoch": 0.3671875, "grad_norm": 0.20110268890857697, "learning_rate": 9.57394378802934e-05, "loss": 0.00528339110314846, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.0053, "step": 94, "tokens/total": 3020416, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 42743} +{"epoch": 0.37109375, "grad_norm": 0.5930032730102539, "learning_rate": 9.56152962916e-05, "loss": 0.030327869579195976, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.03079, "step": 95, "tokens/total": 3048800, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 43140} +{"epoch": 0.375, "grad_norm": 0.8909841179847717, "learning_rate": 9.548946453464296e-05, "loss": 0.016370441764593124, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.01651, "step": 96, "tokens/total": 3081768, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 43625} +{"epoch": 0.37890625, "grad_norm": 0.3624550700187683, "learning_rate": 9.53619478457953e-05, "loss": 0.01848924346268177, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01866, "step": 97, "tokens/total": 3115608, "tokens/train_per_sec_per_gpu": 7.01, "tokens/trainable": 44081} +{"epoch": 0.3828125, "grad_norm": 1.0841931104660034, "learning_rate": 9.523275153154695e-05, "loss": 0.012700332328677177, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.01278, "step": 98, "tokens/total": 3146880, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 44531} +{"epoch": 0.38671875, "grad_norm": 0.4348345994949341, "learning_rate": 9.51018809682839e-05, "loss": 0.019419746473431587, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.01961, "step": 99, "tokens/total": 3180088, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 44978} +{"epoch": 0.390625, "grad_norm": 0.35772058367729187, "learning_rate": 9.49693416020645e-05, "loss": 0.017986875027418137, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.01815, "step": 100, "tokens/total": 3213968, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 45431} +{"epoch": 0.39453125, "grad_norm": 0.3776094913482666, "learning_rate": 9.483513894839276e-05, "loss": 0.009350264444947243, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00939, "step": 101, "tokens/total": 3245976, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 45878} +{"epoch": 0.3984375, "grad_norm": 0.35776615142822266, "learning_rate": 9.469927859198888e-05, "loss": 0.006898871622979641, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00692, "step": 102, "tokens/total": 3278792, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 46365} +{"epoch": 0.40234375, "grad_norm": 0.7095727920532227, "learning_rate": 9.456176618655689e-05, "loss": 0.009382505901157856, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00943, "step": 103, "tokens/total": 3309360, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 46812} +{"epoch": 0.40625, "grad_norm": 0.23843331634998322, "learning_rate": 9.442260745454927e-05, "loss": 0.005987213924527168, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00601, "step": 104, "tokens/total": 3344344, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 47289} +{"epoch": 0.41015625, "grad_norm": 0.2994575500488281, "learning_rate": 9.428180818692884e-05, "loss": 0.0037146934773772955, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00372, "step": 105, "tokens/total": 3378832, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 47790} +{"epoch": 0.4140625, "grad_norm": 0.5120344161987305, "learning_rate": 9.413937424292791e-05, "loss": 0.026622634381055832, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.02698, "step": 106, "tokens/total": 3415416, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 48244} +{"epoch": 0.41796875, "grad_norm": 0.71457439661026, "learning_rate": 9.399531154980424e-05, "loss": 0.018018383532762527, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01818, "step": 107, "tokens/total": 3448120, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 48693} +{"epoch": 0.421875, "grad_norm": 0.5954954624176025, "learning_rate": 9.384962610259455e-05, "loss": 0.018590833991765976, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01876, "step": 108, "tokens/total": 3480800, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.6127908229827881, "learning_rate": 9.370232396386494e-05, "loss": 0.04545528069138527, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.0465, "step": 109, "tokens/total": 3509992, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.49300697445869446, "learning_rate": 9.355341126345868e-05, "loss": 0.02438114956021309, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.02468, "step": 110, "tokens/total": 3542872, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.28760164976119995, "learning_rate": 9.340289419824107e-05, "loss": 0.009904235601425171, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.00995, "step": 111, "tokens/total": 3573896, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.7154160737991333, "learning_rate": 9.325077903184159e-05, "loss": 0.04356271028518677, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.04453, "step": 112, "tokens/total": 3605648, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.25252872705459595, "learning_rate": 9.30970720943932e-05, "loss": 0.026460209861397743, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.02681, "step": 113, "tokens/total": 3638544, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 51388} +{"epoch": 0.4453125, "grad_norm": 0.21903468668460846, "learning_rate": 9.2941779782269e-05, "loss": 0.017834223806858063, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01799, "step": 114, "tokens/total": 3670776, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 51861} +{"epoch": 0.44921875, "grad_norm": 0.28969287872314453, "learning_rate": 9.278490855781596e-05, "loss": 0.01664971560239792, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.01679, "step": 115, "tokens/total": 3701472, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 52300} +{"epoch": 0.453125, "grad_norm": 0.28590354323387146, "learning_rate": 9.262646494908604e-05, "loss": 0.029871530830860138, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.03032, "step": 116, "tokens/total": 3735872, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 52753} +{"epoch": 0.45703125, "grad_norm": 0.4681442677974701, "learning_rate": 9.246645554956457e-05, "loss": 0.041582509875297546, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.04246, "step": 117, "tokens/total": 3770032, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 53221} +{"epoch": 0.4609375, "grad_norm": 0.41021010279655457, "learning_rate": 9.230488701789578e-05, "loss": 0.03508072718977928, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.0357, "step": 118, "tokens/total": 3805264, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 53663} +{"epoch": 0.46484375, "grad_norm": 0.4247853457927704, "learning_rate": 9.214176607760577e-05, "loss": 0.010722275823354721, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.01078, "step": 119, "tokens/total": 3840776, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 54126} +{"epoch": 0.46875, "grad_norm": 0.44968104362487793, "learning_rate": 9.197709951682268e-05, "loss": 0.022827425971627235, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.02309, "step": 120, "tokens/total": 3871376, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 54584} +{"epoch": 0.47265625, "grad_norm": 0.48273965716362, "learning_rate": 9.181089418799428e-05, "loss": 0.019125936552882195, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.01931, "step": 121, "tokens/total": 3904552, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 55067} +{"epoch": 0.4765625, "grad_norm": 0.30431416630744934, "learning_rate": 9.164315700760271e-05, "loss": 0.011825008317828178, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.0119, "step": 122, "tokens/total": 3937344, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 55491} +{"epoch": 0.48046875, "grad_norm": 0.3846381604671478, "learning_rate": 9.147389495587671e-05, "loss": 0.029094792902469635, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.02952, "step": 123, "tokens/total": 3967392, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 55937} +{"epoch": 0.484375, "grad_norm": 0.7719686627388, "learning_rate": 9.130311507650116e-05, "loss": 0.01813952624797821, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.01831, "step": 124, "tokens/total": 3998280, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 56401} +{"epoch": 0.48828125, "grad_norm": 0.2838183343410492, "learning_rate": 9.113082447632394e-05, "loss": 0.013848325237631798, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.01394, "step": 125, "tokens/total": 4029616, "tokens/train_per_sec_per_gpu": 10.35, "tokens/trainable": 56874} +{"epoch": 0.4921875, "grad_norm": 0.4921285808086395, "learning_rate": 9.09570303250602e-05, "loss": 0.008241109549999237, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00828, "step": 126, "tokens/total": 4060368, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 57358} +{"epoch": 0.49609375, "grad_norm": 1.0206093788146973, "learning_rate": 9.078173985499394e-05, "loss": 0.01356454472988844, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.01366, "step": 127, "tokens/total": 4089688, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 57788} +{"epoch": 0.5, "grad_norm": 0.16598685085773468, "learning_rate": 9.060496036067713e-05, "loss": 0.011222724802792072, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.01129, "step": 128, "tokens/total": 4125048, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 58212} +{"epoch": 0.50390625, "grad_norm": 0.37962988018989563, "learning_rate": 9.042669919862615e-05, "loss": 0.018600409850478172, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.01877, "step": 129, "tokens/total": 4156488, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 58676} +{"epoch": 0.5078125, "grad_norm": 1.108361005783081, "learning_rate": 9.024696378701557e-05, "loss": 0.04666392505168915, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.04777, "step": 130, "tokens/total": 4189344, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 59109} +{"epoch": 0.51171875, "grad_norm": 1.0173554420471191, "learning_rate": 9.006576160536948e-05, "loss": 0.021368790417909622, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.0216, "step": 131, "tokens/total": 4220768, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 59558} +{"epoch": 0.515625, "grad_norm": 0.4209519028663635, "learning_rate": 8.988310019425035e-05, "loss": 0.014557933434844017, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.01466, "step": 132, "tokens/total": 4256880, "tokens/train_per_sec_per_gpu": 7.11, "tokens/trainable": 60002} +{"epoch": 0.51953125, "grad_norm": 0.15375781059265137, "learning_rate": 8.969898715494506e-05, "loss": 0.0058281091041862965, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00585, "step": 133, "tokens/total": 4290928, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 60483} +{"epoch": 0.5234375, "grad_norm": 0.34137681126594543, "learning_rate": 8.951343014914869e-05, "loss": 0.024304546415805817, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.0246, "step": 134, "tokens/total": 4325728, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 60949} +{"epoch": 0.52734375, "grad_norm": 0.4883163571357727, "learning_rate": 8.932643689864568e-05, "loss": 0.0045571718364953995, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00457, "step": 135, "tokens/total": 4358848, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 61436} +{"epoch": 0.53125, "grad_norm": 0.43666452169418335, "learning_rate": 8.913801518498845e-05, "loss": 0.012516661547124386, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.0126, "step": 136, "tokens/total": 4393504, "tokens/train_per_sec_per_gpu": 6.83, "tokens/trainable": 61897} +{"epoch": 0.53515625, "grad_norm": 0.799555778503418, "learning_rate": 8.894817284917364e-05, "loss": 0.0104899937286973, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.01055, "step": 137, "tokens/total": 4427912, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 62351} +{"epoch": 0.5390625, "grad_norm": 0.35077399015426636, "learning_rate": 8.875691779131569e-05, "loss": 0.037752632051706314, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.03847, "step": 138, "tokens/total": 4462448, "tokens/train_per_sec_per_gpu": 6.32, "tokens/trainable": 62783} +{"epoch": 0.54296875, "grad_norm": 0.45694610476493835, "learning_rate": 8.856425797031829e-05, "loss": 0.010625991970300674, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01068, "step": 139, "tokens/total": 4495992, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 63225} +{"epoch": 0.546875, "grad_norm": 0.11577413976192474, "learning_rate": 8.837020140354295e-05, "loss": 0.003559976117685437, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00357, "step": 140, "tokens/total": 4529600, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 63690} +{"epoch": 0.55078125, "grad_norm": 0.5336243510246277, "learning_rate": 8.817475616647554e-05, "loss": 0.02105163410305977, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.02127, "step": 141, "tokens/total": 4562536, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 64157} +{"epoch": 0.5546875, "grad_norm": 0.3121412992477417, "learning_rate": 8.797793039239017e-05, "loss": 0.030202940106391907, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.03066, "step": 142, "tokens/total": 4595232, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 64605} +{"epoch": 0.55859375, "grad_norm": 0.20766611397266388, "learning_rate": 8.777973227201069e-05, "loss": 0.006446573883295059, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00647, "step": 143, "tokens/total": 4629712, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 65114} +{"epoch": 0.5625, "grad_norm": 0.2658213973045349, "learning_rate": 8.758017005316988e-05, "loss": 0.010514147579669952, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.01057, "step": 144, "tokens/total": 4661752, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 65539} +{"epoch": 0.56640625, "grad_norm": 0.3977394104003906, "learning_rate": 8.737925204046629e-05, "loss": 0.02311687357723713, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.02339, "step": 145, "tokens/total": 4694416, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 65989} +{"epoch": 0.5703125, "grad_norm": 0.12929683923721313, "learning_rate": 8.717698659491851e-05, "loss": 0.003155242186039686, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00316, "step": 146, "tokens/total": 4728264, "tokens/train_per_sec_per_gpu": 6.95, "tokens/trainable": 66446} +{"epoch": 0.57421875, "grad_norm": 0.5828168392181396, "learning_rate": 8.697338213361735e-05, "loss": 0.03329544514417648, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.03386, "step": 147, "tokens/total": 4758952, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 66894} +{"epoch": 0.578125, "grad_norm": 0.32581159472465515, "learning_rate": 8.676844712937552e-05, "loss": 0.010191806592047215, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.01024, "step": 148, "tokens/total": 4795624, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 67366} +{"epoch": 0.58203125, "grad_norm": 0.4295192062854767, "learning_rate": 8.656219011037509e-05, "loss": 0.020060492679476738, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.02026, "step": 149, "tokens/total": 4828976, "tokens/train_per_sec_per_gpu": 6.54, "tokens/trainable": 67854} +{"epoch": 0.5859375, "grad_norm": 0.3221510350704193, "learning_rate": 8.63546196598125e-05, "loss": 0.01717602089047432, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.01732, "step": 150, "tokens/total": 4862128, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 68302} +{"epoch": 0.58984375, "grad_norm": 0.30732882022857666, "learning_rate": 8.614574441554145e-05, "loss": 0.01806993968784809, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.01823, "step": 151, "tokens/total": 4892760, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 68727} +{"epoch": 0.59375, "grad_norm": 0.2834779918193817, "learning_rate": 8.593557306971349e-05, "loss": 0.010027683340013027, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.01008, "step": 152, "tokens/total": 4925296, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 69182} +{"epoch": 0.59765625, "grad_norm": 1.4284709692001343, "learning_rate": 8.572411436841618e-05, "loss": 0.00602960679680109, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00605, "step": 153, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 69657} +{"epoch": 0.6015625, "grad_norm": 0.2853012681007385, "learning_rate": 8.551137711130922e-05, "loss": 0.008633681572973728, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00867, "step": 154, "tokens/total": 4985520, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 70063} +{"epoch": 0.60546875, "grad_norm": 0.6215614676475525, "learning_rate": 8.529737015125824e-05, "loss": 0.017476221546530724, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.01763, "step": 155, "tokens/total": 5019320, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 70529} +{"epoch": 0.609375, "grad_norm": 0.5780616402626038, "learning_rate": 8.508210239396639e-05, "loss": 0.024776194244623184, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.02509, "step": 156, "tokens/total": 5052056, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 70978} +{"epoch": 0.61328125, "grad_norm": 0.03672261908650398, "learning_rate": 8.486558279760375e-05, "loss": 0.0020099407993257046, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00201, "step": 157, "tokens/total": 5082280, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 71404} +{"epoch": 0.6171875, "grad_norm": 0.29180118441581726, "learning_rate": 8.464782037243449e-05, "loss": 0.005062858108431101, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00508, "step": 158, "tokens/total": 5113912, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 71845} +{"epoch": 0.62109375, "grad_norm": 0.325463205575943, "learning_rate": 8.442882418044202e-05, "loss": 0.008883287198841572, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00892, "step": 159, "tokens/total": 5145200, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 72322} +{"epoch": 0.625, "grad_norm": 0.6973887085914612, "learning_rate": 8.420860333495179e-05, "loss": 0.024614328518509865, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.02492, "step": 160, "tokens/total": 5174928, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 72737} +{"epoch": 0.62890625, "grad_norm": 0.27683722972869873, "learning_rate": 8.398716700025208e-05, "loss": 0.013697904534637928, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.01379, "step": 161, "tokens/total": 5206888, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 73189} +{"epoch": 0.6328125, "grad_norm": 0.4818889796733856, "learning_rate": 8.376452439121266e-05, "loss": 0.03815722092986107, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.03889, "step": 162, "tokens/total": 5236336, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 73640} +{"epoch": 0.63671875, "grad_norm": 0.8420889377593994, "learning_rate": 8.354068477290124e-05, "loss": 0.007552685681730509, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00758, "step": 163, "tokens/total": 5267760, "tokens/train_per_sec_per_gpu": 10.06, "tokens/trainable": 74099} +{"epoch": 0.640625, "grad_norm": 0.19581358134746552, "learning_rate": 8.331565746019807e-05, "loss": 0.011371469125151634, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.01144, "step": 164, "tokens/total": 5298896, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 74557} +{"epoch": 0.64453125, "grad_norm": 1.2153747081756592, "learning_rate": 8.308945181740812e-05, "loss": 0.04914269968867302, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.05037, "step": 165, "tokens/total": 5330792, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 74999} +{"epoch": 0.6484375, "grad_norm": 2.668557643890381, "learning_rate": 8.286207725787153e-05, "loss": 0.010660962201654911, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.01072, "step": 166, "tokens/total": 5364144, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 75444} +{"epoch": 0.65234375, "grad_norm": 0.751739501953125, "learning_rate": 8.263354324357182e-05, "loss": 0.04288284480571747, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.04382, "step": 167, "tokens/total": 5396656, "tokens/train_per_sec_per_gpu": 6.75, "tokens/trainable": 75879} +{"epoch": 0.65625, "grad_norm": 0.19443996250629425, "learning_rate": 8.240385928474219e-05, "loss": 0.012382534332573414, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.01246, "step": 168, "tokens/total": 5427736, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 76293} +{"epoch": 0.66015625, "grad_norm": 1.2003859281539917, "learning_rate": 8.217303493946967e-05, "loss": 0.02118557319045067, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.02141, "step": 169, "tokens/total": 5461752, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 76741} +{"epoch": 0.6640625, "grad_norm": 2.061119318008423, "learning_rate": 8.194107981329746e-05, "loss": 0.014531993307173252, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01464, "step": 170, "tokens/total": 5498688, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 77195} +{"epoch": 0.66796875, "grad_norm": 0.3984575569629669, "learning_rate": 8.170800355882518e-05, "loss": 0.008441315032541752, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00848, "step": 171, "tokens/total": 5530936, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 77662} +{"epoch": 0.671875, "grad_norm": 0.36441314220428467, "learning_rate": 8.147381587530713e-05, "loss": 0.0252967681735754, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.02562, "step": 172, "tokens/total": 5564176, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 78116} +{"epoch": 0.67578125, "grad_norm": 0.39151886105537415, "learning_rate": 8.123852650824877e-05, "loss": 0.031642768532037735, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.03215, "step": 173, "tokens/total": 5596856, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 78599} +{"epoch": 0.6796875, "grad_norm": 0.25767388939857483, "learning_rate": 8.100214524900103e-05, "loss": 0.01829385571181774, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.01846, "step": 174, "tokens/total": 5630016, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 79030} +{"epoch": 0.68359375, "grad_norm": 0.22931413352489471, "learning_rate": 8.076468193435301e-05, "loss": 0.010844210162758827, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.0109, "step": 175, "tokens/total": 5659656, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 79454} +{"epoch": 0.6875, "grad_norm": 0.33504801988601685, "learning_rate": 8.052614644612253e-05, "loss": 0.009478000923991203, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00952, "step": 176, "tokens/total": 5692408, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 79915} +{"epoch": 0.69140625, "grad_norm": 0.13409273326396942, "learning_rate": 8.028654871074489e-05, "loss": 0.0030466816388070583, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00305, "step": 177, "tokens/total": 5724672, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 80356} +{"epoch": 0.6953125, "grad_norm": 0.37993359565734863, "learning_rate": 8.004589869885986e-05, "loss": 0.05233447253704071, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.05373, "step": 178, "tokens/total": 5754384, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 80774} +{"epoch": 0.69921875, "grad_norm": 0.3220449388027191, "learning_rate": 7.980420642489674e-05, "loss": 0.004555789288133383, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00457, "step": 179, "tokens/total": 5789344, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 81247} +{"epoch": 0.703125, "grad_norm": 0.28945472836494446, "learning_rate": 7.95614819466576e-05, "loss": 0.01688355766236782, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.01703, "step": 180, "tokens/total": 5821816, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 81713} +{"epoch": 0.70703125, "grad_norm": 0.8303338885307312, "learning_rate": 7.931773536489872e-05, "loss": 0.02587173879146576, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.02621, "step": 181, "tokens/total": 5854168, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 82158} +{"epoch": 0.7109375, "grad_norm": 0.43549689650535583, "learning_rate": 7.907297682291035e-05, "loss": 0.02028777077794075, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.02049, "step": 182, "tokens/total": 5886064, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 82621} +{"epoch": 0.71484375, "grad_norm": 0.3550703525543213, "learning_rate": 7.882721650609442e-05, "loss": 0.010249704122543335, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.0103, "step": 183, "tokens/total": 5921648, "tokens/train_per_sec_per_gpu": 6.64, "tokens/trainable": 83122} +{"epoch": 0.71875, "grad_norm": 0.2938533127307892, "learning_rate": 7.85804646415409e-05, "loss": 0.015499619767069817, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.01562, "step": 184, "tokens/total": 5954952, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 83578} +{"epoch": 0.72265625, "grad_norm": 0.16341863572597504, "learning_rate": 7.833273149760207e-05, "loss": 0.010686513036489487, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.01074, "step": 185, "tokens/total": 5985432, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 84048} +{"epoch": 0.7265625, "grad_norm": 2.2135605812072754, "learning_rate": 7.808402738346527e-05, "loss": 0.0379524827003479, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.03868, "step": 186, "tokens/total": 6020664, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 84497} +{"epoch": 0.73046875, "grad_norm": 0.14665764570236206, "learning_rate": 7.783436264872382e-05, "loss": 0.003102231537923217, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00311, "step": 187, "tokens/total": 6055024, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 84983} +{"epoch": 0.734375, "grad_norm": 0.19977813959121704, "learning_rate": 7.758374768294647e-05, "loss": 0.007219668943434954, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00725, "step": 188, "tokens/total": 6088296, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 85462} +{"epoch": 0.73828125, "grad_norm": 0.37237226963043213, "learning_rate": 7.733219291524489e-05, "loss": 0.035138219594955444, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.03576, "step": 189, "tokens/total": 6119840, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 85892} +{"epoch": 0.7421875, "grad_norm": 0.492442786693573, "learning_rate": 7.707970881383977e-05, "loss": 0.030790071934461594, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.03127, "step": 190, "tokens/total": 6149880, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 86341} +{"epoch": 0.74609375, "grad_norm": 0.6119093298912048, "learning_rate": 7.682630588562518e-05, "loss": 0.012406530790030956, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01248, "step": 191, "tokens/total": 6182904, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 86772} +{"epoch": 0.75, "grad_norm": 0.2786654531955719, "learning_rate": 7.657199467573129e-05, "loss": 0.016775479540228844, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01692, "step": 192, "tokens/total": 6214144, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 87244} +{"epoch": 0.75390625, "grad_norm": 0.9122756719589233, "learning_rate": 7.631678576708561e-05, "loss": 0.022656021639704704, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.02291, "step": 193, "tokens/total": 6247072, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 87736} +{"epoch": 0.7578125, "grad_norm": 0.14745716750621796, "learning_rate": 7.606068977997255e-05, "loss": 0.013927598483860493, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.01403, "step": 194, "tokens/total": 6281112, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 88214} +{"epoch": 0.76171875, "grad_norm": 0.3804144859313965, "learning_rate": 7.580371737159148e-05, "loss": 0.02501925453543663, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.02533, "step": 195, "tokens/total": 6310200, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 88625} +{"epoch": 0.765625, "grad_norm": 0.17950302362442017, "learning_rate": 7.554587923561324e-05, "loss": 0.005994710139930248, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00601, "step": 196, "tokens/total": 6341792, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 89063} +{"epoch": 0.76953125, "grad_norm": 0.6477283835411072, "learning_rate": 7.528718610173511e-05, "loss": 0.027550119906663895, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.02793, "step": 197, "tokens/total": 6374040, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 89484} +{"epoch": 0.7734375, "grad_norm": 0.41654524207115173, "learning_rate": 7.502764873523431e-05, "loss": 0.016324687749147415, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.01646, "step": 198, "tokens/total": 6405552, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 89922} +{"epoch": 0.77734375, "grad_norm": 0.5638466477394104, "learning_rate": 7.476727793652011e-05, "loss": 0.03229910880327225, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.03283, "step": 199, "tokens/total": 6436016, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 90401} +{"epoch": 0.78125, "grad_norm": 1.1563435792922974, "learning_rate": 7.450608454068415e-05, "loss": 0.03628615662455559, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.03695, "step": 200, "tokens/total": 6468656, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 90842} +{"epoch": 0.78515625, "grad_norm": 0.5204038023948669, "learning_rate": 7.424407941704987e-05, "loss": 0.010748404078185558, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.01081, "step": 201, "tokens/total": 6502728, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 91343} +{"epoch": 0.7890625, "grad_norm": 0.5812761187553406, "learning_rate": 7.398127346871986e-05, "loss": 0.02578561007976532, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.02612, "step": 202, "tokens/total": 6535616, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 91798} +{"epoch": 0.79296875, "grad_norm": 0.33327075839042664, "learning_rate": 7.371767763212238e-05, "loss": 0.00526802521198988, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00528, "step": 203, "tokens/total": 6567088, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 92251} +{"epoch": 0.796875, "grad_norm": 0.41538897156715393, "learning_rate": 7.345330287655617e-05, "loss": 0.027267012745141983, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.02764, "step": 204, "tokens/total": 6600808, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 92714} +{"epoch": 0.80078125, "grad_norm": 0.5190113186836243, "learning_rate": 7.31881602037339e-05, "loss": 0.022373635321855545, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.02263, "step": 205, "tokens/total": 6631384, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 93154} +{"epoch": 0.8046875, "grad_norm": 0.12493547052145004, "learning_rate": 7.29222606473245e-05, "loss": 0.0043559749610722065, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00437, "step": 206, "tokens/total": 6661888, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 93599} +{"epoch": 0.80859375, "grad_norm": 0.7940065860748291, "learning_rate": 7.265561527249383e-05, "loss": 0.05110673978924751, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.05244, "step": 207, "tokens/total": 6693456, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 94036} +{"epoch": 0.8125, "grad_norm": 0.4209924638271332, "learning_rate": 7.238823517544436e-05, "loss": 0.032634228467941284, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.03317, "step": 208, "tokens/total": 6727840, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 94506} +{"epoch": 0.81640625, "grad_norm": 0.8418631553649902, "learning_rate": 7.212013148295333e-05, "loss": 0.007314873859286308, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00734, "step": 209, "tokens/total": 6761704, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 94988} +{"epoch": 0.8203125, "grad_norm": 0.20371142029762268, "learning_rate": 7.185131535190975e-05, "loss": 0.01730574108660221, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.01746, "step": 210, "tokens/total": 6792648, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 95497} +{"epoch": 0.82421875, "grad_norm": 0.2571842670440674, "learning_rate": 7.158179796885005e-05, "loss": 0.013248106464743614, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.01334, "step": 211, "tokens/total": 6824424, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 95948} +{"epoch": 0.828125, "grad_norm": 0.3693482279777527, "learning_rate": 7.131159054949273e-05, "loss": 0.02345692366361618, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.02373, "step": 212, "tokens/total": 6861216, "tokens/train_per_sec_per_gpu": 6.99, "tokens/trainable": 96422} +{"epoch": 0.83203125, "grad_norm": 1.8457674980163574, "learning_rate": 7.104070433827139e-05, "loss": 0.03122623637318611, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.03172, "step": 213, "tokens/total": 6892704, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 96821} +{"epoch": 0.8359375, "grad_norm": 0.14125436544418335, "learning_rate": 7.076915060786705e-05, "loss": 0.012880915775895119, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.01296, "step": 214, "tokens/total": 6924816, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 97263} +{"epoch": 0.83984375, "grad_norm": 0.40359795093536377, "learning_rate": 7.049694065873882e-05, "loss": 0.01780807413160801, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.01797, "step": 215, "tokens/total": 6957536, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 97720} +{"epoch": 0.84375, "grad_norm": 0.5508702397346497, "learning_rate": 7.022408581865382e-05, "loss": 0.018534447997808456, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.01871, "step": 216, "tokens/total": 6991448, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 98194} +{"epoch": 0.84765625, "grad_norm": 0.2615598738193512, "learning_rate": 6.99505974422157e-05, "loss": 0.016732990741729736, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.01687, "step": 217, "tokens/total": 7023712, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 98667} +{"epoch": 0.8515625, "grad_norm": 2.2193048000335693, "learning_rate": 6.967648691039213e-05, "loss": 0.02599203586578369, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.02633, "step": 218, "tokens/total": 7055040, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 99100} +{"epoch": 0.85546875, "grad_norm": 0.18961451947689056, "learning_rate": 6.940176563004123e-05, "loss": 0.007304243743419647, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00733, "step": 219, "tokens/total": 7084944, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 99532} +{"epoch": 0.859375, "grad_norm": 0.18438772857189178, "learning_rate": 6.912644503343682e-05, "loss": 0.00529701542109251, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00531, "step": 220, "tokens/total": 7119264, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 99970} +{"epoch": 0.86328125, "grad_norm": 0.261023610830307, "learning_rate": 6.885053657779273e-05, "loss": 0.019973663613200188, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.02017, "step": 221, "tokens/total": 7150848, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 100442} +{"epoch": 0.8671875, "grad_norm": 0.15022070705890656, "learning_rate": 6.857405174478604e-05, "loss": 0.0037573501467704773, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00376, "step": 222, "tokens/total": 7185176, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 100895} +{"epoch": 0.87109375, "grad_norm": 0.2558874785900116, "learning_rate": 6.82970020400792e-05, "loss": 0.023127881810069084, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.0234, "step": 223, "tokens/total": 7219024, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 101342} +{"epoch": 0.875, "grad_norm": 0.2618688642978668, "learning_rate": 6.801939899284132e-05, "loss": 0.005354383494704962, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00537, "step": 224, "tokens/total": 7253816, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 101848} +{"epoch": 0.87890625, "grad_norm": 0.23234397172927856, "learning_rate": 6.774125415526827e-05, "loss": 0.004253203049302101, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00426, "step": 225, "tokens/total": 7286344, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 102319} +{"epoch": 0.8828125, "grad_norm": 0.23210711777210236, "learning_rate": 6.746257910210214e-05, "loss": 0.013024741783738136, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.01311, "step": 226, "tokens/total": 7317952, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 102769} +{"epoch": 0.88671875, "grad_norm": 0.9405398368835449, "learning_rate": 6.718338543014937e-05, "loss": 0.03316330909729004, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.03372, "step": 227, "tokens/total": 7347336, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 103219} +{"epoch": 0.890625, "grad_norm": 0.10629910975694656, "learning_rate": 6.69036847577983e-05, "loss": 0.003330075182020664, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00334, "step": 228, "tokens/total": 7377592, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 103675} +{"epoch": 0.89453125, "grad_norm": 0.3910622000694275, "learning_rate": 6.662348872453553e-05, "loss": 0.021336494013667107, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.02157, "step": 229, "tokens/total": 7410184, "tokens/train_per_sec_per_gpu": 10.32, "tokens/trainable": 104136} +{"epoch": 0.8984375, "grad_norm": 0.358774334192276, "learning_rate": 6.63428089904618e-05, "loss": 0.012408902868628502, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.01249, "step": 230, "tokens/total": 7443984, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 104588} +{"epoch": 0.90234375, "grad_norm": 0.321442574262619, "learning_rate": 6.60616572358065e-05, "loss": 0.03975185006856918, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.04055, "step": 231, "tokens/total": 7472480, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 104988} +{"epoch": 0.90625, "grad_norm": 0.09658742696046829, "learning_rate": 6.578004516044172e-05, "loss": 0.01401535328477621, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.01411, "step": 232, "tokens/total": 7505992, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 105455} +{"epoch": 0.91015625, "grad_norm": 0.13552908599376678, "learning_rate": 6.549798448339548e-05, "loss": 0.00901111401617527, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00905, "step": 233, "tokens/total": 7541168, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 105918} +{"epoch": 0.9140625, "grad_norm": 0.0958385244011879, "learning_rate": 6.521548694236384e-05, "loss": 0.011448979377746582, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.01151, "step": 234, "tokens/total": 7574208, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 106355} +{"epoch": 0.91796875, "grad_norm": 0.5189796686172485, "learning_rate": 6.493256429322259e-05, "loss": 0.00902932696044445, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00907, "step": 235, "tokens/total": 7607016, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 106786} +{"epoch": 0.921875, "grad_norm": 0.2973494529724121, "learning_rate": 6.464922830953799e-05, "loss": 0.02409461885690689, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.02439, "step": 236, "tokens/total": 7641744, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 107219} +{"epoch": 0.92578125, "grad_norm": 0.3081362843513489, "learning_rate": 6.436549078207688e-05, "loss": 0.019756125286221504, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.01995, "step": 237, "tokens/total": 7673096, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 107679} +{"epoch": 0.9296875, "grad_norm": 0.6635058522224426, "learning_rate": 6.408136351831592e-05, "loss": 0.01119239255785942, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.01126, "step": 238, "tokens/total": 7705344, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 108119} +{"epoch": 0.93359375, "grad_norm": 0.2612011432647705, "learning_rate": 6.379685834195036e-05, "loss": 0.010111300274729729, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.01016, "step": 239, "tokens/total": 7740240, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 108626} +{"epoch": 0.9375, "grad_norm": 0.2098863571882248, "learning_rate": 6.351198709240186e-05, "loss": 0.02142259106040001, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.02165, "step": 240, "tokens/total": 7775392, "tokens/train_per_sec_per_gpu": 6.99, "tokens/trainable": 109079} +{"epoch": 0.94140625, "grad_norm": 0.10193968564271927, "learning_rate": 6.32267616243259e-05, "loss": 0.006294456776231527, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00631, "step": 241, "tokens/total": 7807296, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 109542} +{"epoch": 0.9453125, "grad_norm": 0.2935059666633606, "learning_rate": 6.294119380711849e-05, "loss": 0.01695157401263714, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.0171, "step": 242, "tokens/total": 7838200, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 109969} +{"epoch": 0.94921875, "grad_norm": 0.3051242232322693, "learning_rate": 6.265529552442209e-05, "loss": 0.03947668895125389, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.04027, "step": 243, "tokens/total": 7870936, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 110431} +{"epoch": 0.953125, "grad_norm": 0.19432295858860016, "learning_rate": 6.236907867363127e-05, "loss": 0.005742155946791172, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00576, "step": 244, "tokens/total": 7902512, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 110877} +{"epoch": 0.95703125, "grad_norm": 0.051487356424331665, "learning_rate": 6.208255516539749e-05, "loss": 0.0032077583018690348, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00321, "step": 245, "tokens/total": 7933224, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 111350} +{"epoch": 0.9609375, "grad_norm": 0.07877344638109207, "learning_rate": 6.179573692313344e-05, "loss": 0.01269505638629198, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.01278, "step": 246, "tokens/total": 7968456, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 111798} +{"epoch": 0.96484375, "grad_norm": 0.3711574375629425, "learning_rate": 6.150863588251694e-05, "loss": 0.020388402044773102, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.0206, "step": 247, "tokens/total": 8003592, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 112264} +{"epoch": 0.96875, "grad_norm": 0.5861304402351379, "learning_rate": 6.122126399099419e-05, "loss": 0.014131024479866028, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.01423, "step": 248, "tokens/total": 8040048, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 112772} +{"epoch": 0.97265625, "grad_norm": 0.21137702465057373, "learning_rate": 6.0933633207282615e-05, "loss": 0.024793460965156555, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.0251, "step": 249, "tokens/total": 8074000, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 113278} +{"epoch": 0.9765625, "grad_norm": 0.21779443323612213, "learning_rate": 6.064575550087316e-05, "loss": 0.026697520166635513, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.02706, "step": 250, "tokens/total": 8108304, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 113716} +{"epoch": 0.98046875, "grad_norm": 2.8734798431396484, "learning_rate": 6.0357642851532245e-05, "loss": 0.006318456493318081, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00634, "step": 251, "tokens/total": 8143464, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 114207} +{"epoch": 0.984375, "grad_norm": 0.1974196583032608, "learning_rate": 6.0069307248803294e-05, "loss": 0.012006456032395363, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.01208, "step": 252, "tokens/total": 8175576, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 114628} +{"epoch": 0.98828125, "grad_norm": 0.11235077679157257, "learning_rate": 5.9780760691507635e-05, "loss": 0.00962769240140915, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00967, "step": 253, "tokens/total": 8205528, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 115060} +{"epoch": 0.9921875, "grad_norm": 0.32756802439689636, "learning_rate": 5.9492015187245334e-05, "loss": 0.011568691581487656, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.01164, "step": 254, "tokens/total": 8239424, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 115514} +{"epoch": 0.99609375, "grad_norm": 0.16435463726520538, "learning_rate": 5.920308275189541e-05, "loss": 0.028547395020723343, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.02896, "step": 255, "tokens/total": 8270392, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 115915} +{"epoch": 1.0, "grad_norm": 0.11991561949253082, "learning_rate": 5.8913975409115874e-05, "loss": 0.0026995930820703506, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.0027, "step": 256, "tokens/total": 8302096, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 116378} +{"epoch": 1.00390625, "grad_norm": 0.023384150117635727, "learning_rate": 5.8624705189843395e-05, "loss": 0.0010946302209049463, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.0011, "step": 257, "tokens/total": 8337064, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 116865} +{"epoch": 1.0078125, "grad_norm": 0.07135042548179626, "learning_rate": 5.833528413179249e-05, "loss": 0.00851083267480135, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00855, "step": 258, "tokens/total": 8372840, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 117363} +{"epoch": 1.01171875, "grad_norm": 0.13214389979839325, "learning_rate": 5.80457242789548e-05, "loss": 0.009941112250089645, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00999, "step": 259, "tokens/total": 8409728, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 117860} +{"epoch": 1.015625, "grad_norm": 0.36253926157951355, "learning_rate": 5.77560376810977e-05, "loss": 0.025427334010601044, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.02575, "step": 260, "tokens/total": 8442296, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 118324} +{"epoch": 1.01953125, "grad_norm": 0.15676242113113403, "learning_rate": 5.7466236393263005e-05, "loss": 0.011946442537009716, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.01202, "step": 261, "tokens/total": 8474568, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 118770} +{"epoch": 1.0234375, "grad_norm": 0.2707855701446533, "learning_rate": 5.717633247526522e-05, "loss": 0.024459855630993843, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.02476, "step": 262, "tokens/total": 8505520, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 119206} +{"epoch": 1.02734375, "grad_norm": 0.1849583387374878, "learning_rate": 5.688633799118971e-05, "loss": 0.017461063340306282, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.01761, "step": 263, "tokens/total": 8535064, "tokens/train_per_sec_per_gpu": 7.21, "tokens/trainable": 119610} +{"epoch": 1.03125, "grad_norm": 0.20750857889652252, "learning_rate": 5.659626500889066e-05, "loss": 0.004097632132470608, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00411, "step": 264, "tokens/total": 8563600, "tokens/train_per_sec_per_gpu": 6.91, "tokens/trainable": 120080} +{"epoch": 1.03515625, "grad_norm": 0.06770914793014526, "learning_rate": 5.6306125599488905e-05, "loss": 0.010814931243658066, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.01087, "step": 265, "tokens/total": 8592648, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 120519} +{"epoch": 1.0390625, "grad_norm": 0.28017815947532654, "learning_rate": 5.601593183686955e-05, "loss": 0.005327209830284119, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00534, "step": 266, "tokens/total": 8627328, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 120995} +{"epoch": 1.04296875, "grad_norm": 0.388915091753006, "learning_rate": 5.572569579717961e-05, "loss": 0.02028101682662964, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.02049, "step": 267, "tokens/total": 8659760, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 121459} +{"epoch": 1.046875, "grad_norm": 0.1730630248785019, "learning_rate": 5.543542955832538e-05, "loss": 0.004020120482891798, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00403, "step": 268, "tokens/total": 8696968, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 121902} +{"epoch": 1.05078125, "grad_norm": 0.1799696534872055, "learning_rate": 5.514514519946986e-05, "loss": 0.015057987533509731, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01517, "step": 269, "tokens/total": 8735392, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 122399} +{"epoch": 1.0546875, "grad_norm": 0.2448037713766098, "learning_rate": 5.485485480053015e-05, "loss": 0.0195162370800972, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.01971, "step": 270, "tokens/total": 8769840, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 122804} +{"epoch": 1.05859375, "grad_norm": 0.1312880665063858, "learning_rate": 5.4564570441674645e-05, "loss": 0.016576452180743217, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.01671, "step": 271, "tokens/total": 8800648, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 123244} +{"epoch": 1.0625, "grad_norm": 0.17985928058624268, "learning_rate": 5.42743042028204e-05, "loss": 0.017611032351851463, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.01777, "step": 272, "tokens/total": 8832480, "tokens/train_per_sec_per_gpu": 6.68, "tokens/trainable": 123683} +{"epoch": 1.06640625, "grad_norm": 0.6411257386207581, "learning_rate": 5.3984068163130464e-05, "loss": 0.013030522502958775, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.01312, "step": 273, "tokens/total": 8864840, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 124153} +{"epoch": 1.0703125, "grad_norm": 0.16655303537845612, "learning_rate": 5.369387440051111e-05, "loss": 0.0035718497820198536, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00358, "step": 274, "tokens/total": 8897272, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 124616} +{"epoch": 1.07421875, "grad_norm": 0.42606455087661743, "learning_rate": 5.340373499110935e-05, "loss": 0.015140683390200138, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.01526, "step": 275, "tokens/total": 8930688, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 125109} +{"epoch": 1.078125, "grad_norm": 0.0429648756980896, "learning_rate": 5.3113662008810304e-05, "loss": 0.0018917323322966695, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00189, "step": 276, "tokens/total": 8967216, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 125591} +{"epoch": 1.08203125, "grad_norm": 0.22177483141422272, "learning_rate": 5.282366752473479e-05, "loss": 0.007883564569056034, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00791, "step": 277, "tokens/total": 9001368, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 126065} +{"epoch": 1.0859375, "grad_norm": 0.4673311412334442, "learning_rate": 5.2533763606737005e-05, "loss": 0.023375801742076874, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.02365, "step": 278, "tokens/total": 9036808, "tokens/train_per_sec_per_gpu": 6.77, "tokens/trainable": 126480} +{"epoch": 1.08984375, "grad_norm": 0.09221015870571136, "learning_rate": 5.224396231890232e-05, "loss": 0.003989626187831163, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.004, "step": 279, "tokens/total": 9067840, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 126929} +{"epoch": 1.09375, "grad_norm": 0.5374587774276733, "learning_rate": 5.195427572104522e-05, "loss": 0.025742007419466972, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.02608, "step": 280, "tokens/total": 9098936, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 127366} +{"epoch": 1.09765625, "grad_norm": 0.06573054939508438, "learning_rate": 5.166471586820751e-05, "loss": 0.008180230855941772, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00821, "step": 281, "tokens/total": 9133264, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 127831} +{"epoch": 1.1015625, "grad_norm": 0.4904385507106781, "learning_rate": 5.1375294810156615e-05, "loss": 0.01513670478016138, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.01525, "step": 282, "tokens/total": 9165008, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 128278} +{"epoch": 1.10546875, "grad_norm": 0.08064861595630646, "learning_rate": 5.1086024590884144e-05, "loss": 0.0028609393630176783, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00287, "step": 283, "tokens/total": 9197952, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 128727} +{"epoch": 1.109375, "grad_norm": 0.04141188785433769, "learning_rate": 5.079691724810461e-05, "loss": 0.0020744693465530872, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00208, "step": 284, "tokens/total": 9232576, "tokens/train_per_sec_per_gpu": 6.98, "tokens/trainable": 129162} +{"epoch": 1.11328125, "grad_norm": 0.6608247756958008, "learning_rate": 5.0507984812754684e-05, "loss": 0.004125798586755991, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00413, "step": 285, "tokens/total": 9264568, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 129659} +{"epoch": 1.1171875, "grad_norm": 0.35650819540023804, "learning_rate": 5.021923930849237e-05, "loss": 0.014661246910691261, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.01477, "step": 286, "tokens/total": 9296544, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 130145} +{"epoch": 1.12109375, "grad_norm": 0.03451412916183472, "learning_rate": 4.99306927511967e-05, "loss": 0.0016670986078679562, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00167, "step": 287, "tokens/total": 9327936, "tokens/train_per_sec_per_gpu": 10.25, "tokens/trainable": 130585} +{"epoch": 1.125, "grad_norm": 0.06517993658781052, "learning_rate": 4.964235714846775e-05, "loss": 0.00200478988699615, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00201, "step": 288, "tokens/total": 9357984, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 131005} +{"epoch": 1.12890625, "grad_norm": 0.016634000465273857, "learning_rate": 4.9354244499126866e-05, "loss": 0.0005826482665725052, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00058, "step": 289, "tokens/total": 9388664, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 131471} +{"epoch": 1.1328125, "grad_norm": 0.02379985712468624, "learning_rate": 4.90663667927174e-05, "loss": 0.0010717754485085607, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00107, "step": 290, "tokens/total": 9420504, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 131920} +{"epoch": 1.13671875, "grad_norm": 0.13503068685531616, "learning_rate": 4.877873600900581e-05, "loss": 0.014873167499899864, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.01498, "step": 291, "tokens/total": 9452664, "tokens/train_per_sec_per_gpu": 6.19, "tokens/trainable": 132379} +{"epoch": 1.140625, "grad_norm": 0.3385327160358429, "learning_rate": 4.849136411748306e-05, "loss": 0.0011530605843290687, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00115, "step": 292, "tokens/total": 9488896, "tokens/train_per_sec_per_gpu": 7.67, "tokens/trainable": 132851} +{"epoch": 1.14453125, "grad_norm": 0.15472468733787537, "learning_rate": 4.8204263076866574e-05, "loss": 0.011289725080132484, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01135, "step": 293, "tokens/total": 9522112, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 133351} +{"epoch": 1.1484375, "grad_norm": 0.5044004321098328, "learning_rate": 4.791744483460251e-05, "loss": 0.01499580405652523, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.01511, "step": 294, "tokens/total": 9555632, "tokens/train_per_sec_per_gpu": 6.47, "tokens/trainable": 133784} +{"epoch": 1.15234375, "grad_norm": 0.014520504511892796, "learning_rate": 4.7630921326368736e-05, "loss": 0.0005424970295280218, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00054, "step": 295, "tokens/total": 9590904, "tokens/train_per_sec_per_gpu": 6.31, "tokens/trainable": 134209} +{"epoch": 1.15625, "grad_norm": 0.1988109052181244, "learning_rate": 4.7344704475577916e-05, "loss": 0.0019340838771313429, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00194, "step": 296, "tokens/total": 9620288, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 134645} +{"epoch": 1.16015625, "grad_norm": 0.4845624566078186, "learning_rate": 4.705880619288153e-05, "loss": 0.006520588882267475, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00654, "step": 297, "tokens/total": 9651984, "tokens/train_per_sec_per_gpu": 6.98, "tokens/trainable": 135047} +{"epoch": 1.1640625, "grad_norm": 0.4207718074321747, "learning_rate": 4.677323837567412e-05, "loss": 0.006265872623771429, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00629, "step": 298, "tokens/total": 9681368, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 135474} +{"epoch": 1.16796875, "grad_norm": 0.06347829848527908, "learning_rate": 4.6488012907598146e-05, "loss": 0.0008359928615391254, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00084, "step": 299, "tokens/total": 9712872, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 135948} +{"epoch": 1.171875, "grad_norm": 0.4166624844074249, "learning_rate": 4.620314165804964e-05, "loss": 0.00670661311596632, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00673, "step": 300, "tokens/total": 9745184, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 136421} +{"epoch": 1.17578125, "grad_norm": 0.1854124665260315, "learning_rate": 4.591863648168407e-05, "loss": 0.015256349928677082, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01537, "step": 301, "tokens/total": 9778928, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 136886} +{"epoch": 1.1796875, "grad_norm": 0.06738762557506561, "learning_rate": 4.5634509217923135e-05, "loss": 0.0012002362636849284, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.0012, "step": 302, "tokens/total": 9810752, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 137321} +{"epoch": 1.18359375, "grad_norm": 0.17503292858600616, "learning_rate": 4.535077169046201e-05, "loss": 0.002777794608846307, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00278, "step": 303, "tokens/total": 9843672, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 137758} +{"epoch": 1.1875, "grad_norm": 0.5695635080337524, "learning_rate": 4.506743570677743e-05, "loss": 0.027430366724729538, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.02781, "step": 304, "tokens/total": 9873752, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 138190} +{"epoch": 1.19140625, "grad_norm": 0.15802061557769775, "learning_rate": 4.478451305763618e-05, "loss": 0.0027470728382468224, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00275, "step": 305, "tokens/total": 9906272, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 138650} +{"epoch": 1.1953125, "grad_norm": 0.07565832883119583, "learning_rate": 4.450201551660454e-05, "loss": 0.0013971275184303522, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.0014, "step": 306, "tokens/total": 9936296, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 139082} +{"epoch": 1.19921875, "grad_norm": 0.3671417534351349, "learning_rate": 4.4219954839558276e-05, "loss": 0.008772294968366623, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00881, "step": 307, "tokens/total": 9967720, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 139523} +{"epoch": 1.203125, "grad_norm": 0.39337456226348877, "learning_rate": 4.393834276419352e-05, "loss": 0.00830507930368185, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.00834, "step": 308, "tokens/total": 9997176, "tokens/train_per_sec_per_gpu": 10.61, "tokens/trainable": 140010} +{"epoch": 1.20703125, "grad_norm": 0.5578709244728088, "learning_rate": 4.36571910095382e-05, "loss": 0.05233357846736908, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.05373, "step": 309, "tokens/total": 10031872, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 140454} +{"epoch": 1.2109375, "grad_norm": 0.6082148551940918, "learning_rate": 4.337651127546448e-05, "loss": 0.023316675797104836, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.02359, "step": 310, "tokens/total": 10065240, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 140905} +{"epoch": 1.21484375, "grad_norm": 0.21418070793151855, "learning_rate": 4.3096315242201736e-05, "loss": 0.01955779269337654, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.01975, "step": 311, "tokens/total": 10096640, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 141339} +{"epoch": 1.21875, "grad_norm": 0.03899383544921875, "learning_rate": 4.2816614569850635e-05, "loss": 0.001552040921524167, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00155, "step": 312, "tokens/total": 10130192, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 141778} +{"epoch": 1.22265625, "grad_norm": 0.10126819461584091, "learning_rate": 4.2537420897897864e-05, "loss": 0.010149841196835041, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.0102, "step": 313, "tokens/total": 10164248, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 142255} +{"epoch": 1.2265625, "grad_norm": 0.24938157200813293, "learning_rate": 4.225874584473174e-05, "loss": 0.016665706411004066, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.01681, "step": 314, "tokens/total": 10194664, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 142693} +{"epoch": 1.23046875, "grad_norm": 0.14164035022258759, "learning_rate": 4.19806010071587e-05, "loss": 0.015035009942948818, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.01515, "step": 315, "tokens/total": 10227768, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 143157} +{"epoch": 1.234375, "grad_norm": 0.12251988798379898, "learning_rate": 4.170299795992081e-05, "loss": 0.013141334056854248, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.01323, "step": 316, "tokens/total": 10260464, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 143603} +{"epoch": 1.23828125, "grad_norm": 0.08062422275543213, "learning_rate": 4.142594825521398e-05, "loss": 0.004540363792330027, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00455, "step": 317, "tokens/total": 10297528, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 144084} +{"epoch": 1.2421875, "grad_norm": 0.1759331077337265, "learning_rate": 4.114946342220728e-05, "loss": 0.012058835476636887, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01213, "step": 318, "tokens/total": 10331448, "tokens/train_per_sec_per_gpu": 6.86, "tokens/trainable": 144532} +{"epoch": 1.24609375, "grad_norm": 0.13066260516643524, "learning_rate": 4.087355496656321e-05, "loss": 0.006143001839518547, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00616, "step": 319, "tokens/total": 10363152, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 145004} +{"epoch": 1.25, "grad_norm": 0.11987114697694778, "learning_rate": 4.05982343699588e-05, "loss": 0.011102660559117794, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.01116, "step": 320, "tokens/total": 10397832, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 145498} +{"epoch": 1.25390625, "grad_norm": 0.11908702552318573, "learning_rate": 4.0323513089607876e-05, "loss": 0.005553742870688438, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00557, "step": 321, "tokens/total": 10431416, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 145965} +{"epoch": 1.2578125, "grad_norm": 0.40266573429107666, "learning_rate": 4.004940255778431e-05, "loss": 0.010549271479249, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01061, "step": 322, "tokens/total": 10467384, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 146409} +{"epoch": 1.26171875, "grad_norm": 0.4372098445892334, "learning_rate": 3.977591418134619e-05, "loss": 0.011345970444381237, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01141, "step": 323, "tokens/total": 10500792, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 146849} +{"epoch": 1.265625, "grad_norm": 0.1920895278453827, "learning_rate": 3.95030593412612e-05, "loss": 0.011139130219817162, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.0112, "step": 324, "tokens/total": 10532592, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 147272} +{"epoch": 1.26953125, "grad_norm": 0.3652326762676239, "learning_rate": 3.923084939213296e-05, "loss": 0.019656235352158546, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.01985, "step": 325, "tokens/total": 10562392, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 147694} +{"epoch": 1.2734375, "grad_norm": 0.13658978044986725, "learning_rate": 3.895929566172861e-05, "loss": 0.0048308661207556725, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00484, "step": 326, "tokens/total": 10595416, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 148158} +{"epoch": 1.27734375, "grad_norm": 0.4624806344509125, "learning_rate": 3.868840945050728e-05, "loss": 0.0070344023406505585, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00706, "step": 327, "tokens/total": 10630504, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 148590} +{"epoch": 1.28125, "grad_norm": 0.20122551918029785, "learning_rate": 3.841820203114995e-05, "loss": 0.015508035197854042, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.01563, "step": 328, "tokens/total": 10661632, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 149037} +{"epoch": 1.28515625, "grad_norm": 0.040351007133722305, "learning_rate": 3.814868464809027e-05, "loss": 0.0022655136417597532, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00227, "step": 329, "tokens/total": 10691544, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 149439} +{"epoch": 1.2890625, "grad_norm": 0.33991917967796326, "learning_rate": 3.787986851704667e-05, "loss": 0.020492341369390488, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.0207, "step": 330, "tokens/total": 10724768, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 149869} +{"epoch": 1.29296875, "grad_norm": 0.043468181043863297, "learning_rate": 3.7611764824555654e-05, "loss": 0.0009273076429963112, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00093, "step": 331, "tokens/total": 10756008, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 150343} +{"epoch": 1.296875, "grad_norm": 0.44356921315193176, "learning_rate": 3.734438472750619e-05, "loss": 0.013238856568932533, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.01333, "step": 332, "tokens/total": 10793152, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 150814} +{"epoch": 1.30078125, "grad_norm": 0.2705346941947937, "learning_rate": 3.707773935267552e-05, "loss": 0.0063155824318528175, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00634, "step": 333, "tokens/total": 10827624, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 151300} +{"epoch": 1.3046875, "grad_norm": 0.04020919278264046, "learning_rate": 3.68118397962661e-05, "loss": 0.0012791571207344532, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00128, "step": 334, "tokens/total": 10859712, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 151708} +{"epoch": 1.30859375, "grad_norm": 0.0606219656765461, "learning_rate": 3.654669712344384e-05, "loss": 0.0014162871520966291, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00142, "step": 335, "tokens/total": 10888792, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 152164} +{"epoch": 1.3125, "grad_norm": 0.06618084013462067, "learning_rate": 3.628232236787763e-05, "loss": 0.0016660630935803056, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00167, "step": 336, "tokens/total": 10922968, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 152603} +{"epoch": 1.31640625, "grad_norm": 0.5435449481010437, "learning_rate": 3.6018726531280144e-05, "loss": 0.007332713343203068, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00736, "step": 337, "tokens/total": 10958000, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 153094} +{"epoch": 1.3203125, "grad_norm": 0.7441946864128113, "learning_rate": 3.575592058295017e-05, "loss": 0.019040189683437347, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.01922, "step": 338, "tokens/total": 10989872, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 153573} +{"epoch": 1.32421875, "grad_norm": 0.7406004667282104, "learning_rate": 3.549391545931585e-05, "loss": 0.0035059102810919285, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00351, "step": 339, "tokens/total": 11020784, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 154033} +{"epoch": 1.328125, "grad_norm": 0.5398491621017456, "learning_rate": 3.5232722063479914e-05, "loss": 0.01695505529642105, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.0171, "step": 340, "tokens/total": 11053240, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 154482} +{"epoch": 1.33203125, "grad_norm": 0.08881665766239166, "learning_rate": 3.49723512647657e-05, "loss": 0.0010589266894385219, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00106, "step": 341, "tokens/total": 11089040, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 154982} +{"epoch": 1.3359375, "grad_norm": 0.4692009687423706, "learning_rate": 3.471281389826491e-05, "loss": 0.006375217344611883, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.0064, "step": 342, "tokens/total": 11119672, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 155387} +{"epoch": 1.33984375, "grad_norm": 0.27360999584198, "learning_rate": 3.4454120764386764e-05, "loss": 0.005366616882383823, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00538, "step": 343, "tokens/total": 11155040, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 155842} +{"epoch": 1.34375, "grad_norm": 0.273283451795578, "learning_rate": 3.4196282628408526e-05, "loss": 0.008576147258281708, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00861, "step": 344, "tokens/total": 11185952, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 156287} +{"epoch": 1.34765625, "grad_norm": 0.038071420043706894, "learning_rate": 3.3939310220027456e-05, "loss": 0.001105990377254784, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00111, "step": 345, "tokens/total": 11219272, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 156782} +{"epoch": 1.3515625, "grad_norm": 0.10952558368444443, "learning_rate": 3.3683214232914404e-05, "loss": 0.0011766317766159773, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00118, "step": 346, "tokens/total": 11253424, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 157255} +{"epoch": 1.35546875, "grad_norm": 0.036292582750320435, "learning_rate": 3.342800532426873e-05, "loss": 0.000535947154276073, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00054, "step": 347, "tokens/total": 11286216, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 157721} +{"epoch": 1.359375, "grad_norm": 0.07606372237205505, "learning_rate": 3.317369411437484e-05, "loss": 0.0016372681129723787, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00164, "step": 348, "tokens/total": 11319480, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 158189} +{"epoch": 1.36328125, "grad_norm": 0.042027659714221954, "learning_rate": 3.292029118616024e-05, "loss": 0.001131725963205099, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00113, "step": 349, "tokens/total": 11350472, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 158612} +{"epoch": 1.3671875, "grad_norm": 0.07330547273159027, "learning_rate": 3.266780708475511e-05, "loss": 0.0018942924216389656, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.0019, "step": 350, "tokens/total": 11383088, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 159090} +{"epoch": 1.37109375, "grad_norm": 0.2684687376022339, "learning_rate": 3.241625231705354e-05, "loss": 0.008634097874164581, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00867, "step": 351, "tokens/total": 11418392, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 159549} +{"epoch": 1.375, "grad_norm": 0.20435288548469543, "learning_rate": 3.216563735127618e-05, "loss": 0.0016702960710972548, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00167, "step": 352, "tokens/total": 11449784, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 160026} +{"epoch": 1.37890625, "grad_norm": 0.764988362789154, "learning_rate": 3.191597261653475e-05, "loss": 0.0241300780326128, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.02442, "step": 353, "tokens/total": 11484240, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 160487} +{"epoch": 1.3828125, "grad_norm": 0.07602517306804657, "learning_rate": 3.166726850239794e-05, "loss": 0.0011442736722528934, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00114, "step": 354, "tokens/total": 11513360, "tokens/train_per_sec_per_gpu": 10.79, "tokens/trainable": 160935} +{"epoch": 1.38671875, "grad_norm": 0.010520228184759617, "learning_rate": 3.141953535845912e-05, "loss": 0.00041890228749252856, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00042, "step": 355, "tokens/total": 11543112, "tokens/train_per_sec_per_gpu": 10.09, "tokens/trainable": 161385} +{"epoch": 1.390625, "grad_norm": 0.457235723733902, "learning_rate": 3.11727834939056e-05, "loss": 0.010767950676381588, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.01083, "step": 356, "tokens/total": 11573464, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 161849} +{"epoch": 1.39453125, "grad_norm": 0.24651657044887543, "learning_rate": 3.092702317708967e-05, "loss": 0.006999955978244543, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00702, "step": 357, "tokens/total": 11604896, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 162306} +{"epoch": 1.3984375, "grad_norm": 0.2988249659538269, "learning_rate": 3.0682264635101276e-05, "loss": 0.007145279087126255, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00717, "step": 358, "tokens/total": 11636368, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 162759} +{"epoch": 1.40234375, "grad_norm": 0.25000572204589844, "learning_rate": 3.0438518053342407e-05, "loss": 0.006719847209751606, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00674, "step": 359, "tokens/total": 11670216, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 163230} +{"epoch": 1.40625, "grad_norm": 0.07347138226032257, "learning_rate": 3.0195793575103266e-05, "loss": 0.0007713790400885046, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00077, "step": 360, "tokens/total": 11707816, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 163694} +{"epoch": 1.41015625, "grad_norm": 0.14972925186157227, "learning_rate": 2.9954101301140146e-05, "loss": 0.009377919137477875, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00942, "step": 361, "tokens/total": 11742584, "tokens/train_per_sec_per_gpu": 6.78, "tokens/trainable": 164144} +{"epoch": 1.4140625, "grad_norm": 0.09731962531805038, "learning_rate": 2.9713451289255123e-05, "loss": 0.002923729130998254, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00293, "step": 362, "tokens/total": 11770384, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 164585} +{"epoch": 1.41796875, "grad_norm": 0.22691254317760468, "learning_rate": 2.9473853553877484e-05, "loss": 0.0031339076813310385, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00314, "step": 363, "tokens/total": 11803208, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 165059} +{"epoch": 1.421875, "grad_norm": 0.22092832624912262, "learning_rate": 2.9235318065647e-05, "loss": 0.007026852108538151, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00705, "step": 364, "tokens/total": 11840480, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 165519} +{"epoch": 1.42578125, "grad_norm": 0.1304856687784195, "learning_rate": 2.8997854750998964e-05, "loss": 0.0015943726757541299, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.0016, "step": 365, "tokens/total": 11875544, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 165984} +{"epoch": 1.4296875, "grad_norm": 0.11849988996982574, "learning_rate": 2.8761473491751258e-05, "loss": 0.002258036518469453, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00226, "step": 366, "tokens/total": 11907656, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 166419} +{"epoch": 1.43359375, "grad_norm": 0.040067367255687714, "learning_rate": 2.8526184124692883e-05, "loss": 0.0010174453491345048, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00102, "step": 367, "tokens/total": 11941904, "tokens/train_per_sec_per_gpu": 6.3, "tokens/trainable": 166857} +{"epoch": 1.4375, "grad_norm": 0.23579032719135284, "learning_rate": 2.829199644117484e-05, "loss": 0.006751311011612415, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00677, "step": 368, "tokens/total": 11973624, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 167267} +{"epoch": 1.44140625, "grad_norm": 0.4901098608970642, "learning_rate": 2.8058920186702553e-05, "loss": 0.0026589068584144115, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00266, "step": 369, "tokens/total": 12010360, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 167740} +{"epoch": 1.4453125, "grad_norm": 0.027983421459794044, "learning_rate": 2.782696506053033e-05, "loss": 0.0006824896554462612, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00068, "step": 370, "tokens/total": 12044544, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 168172} +{"epoch": 1.44921875, "grad_norm": 0.11074771732091904, "learning_rate": 2.7596140715257824e-05, "loss": 0.002904032589867711, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00291, "step": 371, "tokens/total": 12075792, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 168637} +{"epoch": 1.453125, "grad_norm": 0.12958280742168427, "learning_rate": 2.7366456756428184e-05, "loss": 0.0015094542177394032, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00151, "step": 372, "tokens/total": 12108088, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 169094} +{"epoch": 1.45703125, "grad_norm": 0.422427237033844, "learning_rate": 2.7137922742128486e-05, "loss": 0.005157063249498606, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00517, "step": 373, "tokens/total": 12140888, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 169593} +{"epoch": 1.4609375, "grad_norm": 0.17947983741760254, "learning_rate": 2.691054818259188e-05, "loss": 0.0028895798604935408, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00289, "step": 374, "tokens/total": 12174560, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 170066} +{"epoch": 1.46484375, "grad_norm": 0.1037321463227272, "learning_rate": 2.6684342539801933e-05, "loss": 0.001273718778975308, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00127, "step": 375, "tokens/total": 12204848, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 170518} +{"epoch": 1.46875, "grad_norm": 0.0256977416574955, "learning_rate": 2.645931522709877e-05, "loss": 0.0004936883924528956, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00049, "step": 376, "tokens/total": 12238472, "tokens/train_per_sec_per_gpu": 10.59, "tokens/trainable": 170998} +{"epoch": 1.47265625, "grad_norm": 0.25571614503860474, "learning_rate": 2.6235475608787365e-05, "loss": 0.0019885976798832417, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00199, "step": 377, "tokens/total": 12271024, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 171459} +{"epoch": 1.4765625, "grad_norm": 0.8825758099555969, "learning_rate": 2.6012832999747916e-05, "loss": 0.01183762401342392, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.01191, "step": 378, "tokens/total": 12301312, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 171905} +{"epoch": 1.48046875, "grad_norm": 0.31953075528144836, "learning_rate": 2.579139666504821e-05, "loss": 0.01435739267617464, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01446, "step": 379, "tokens/total": 12337072, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 172351} +{"epoch": 1.484375, "grad_norm": 0.09850876778364182, "learning_rate": 2.557117581955798e-05, "loss": 0.0014888396253809333, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00149, "step": 380, "tokens/total": 12368000, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 172817} +{"epoch": 1.48828125, "grad_norm": 0.7366056442260742, "learning_rate": 2.5352179627565532e-05, "loss": 0.01835010014474392, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.01852, "step": 381, "tokens/total": 12399880, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 173258} +{"epoch": 1.4921875, "grad_norm": 0.022171836346387863, "learning_rate": 2.5134417202396277e-05, "loss": 0.0005451083998195827, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00055, "step": 382, "tokens/total": 12432288, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 173665} +{"epoch": 1.49609375, "grad_norm": 0.1049119383096695, "learning_rate": 2.491789760603361e-05, "loss": 0.0016569305444136262, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00166, "step": 383, "tokens/total": 12465592, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 174095} +{"epoch": 1.5, "grad_norm": 0.015806440263986588, "learning_rate": 2.4702629848741764e-05, "loss": 0.00034984052763320506, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00035, "step": 384, "tokens/total": 12498200, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 174550} +{"epoch": 1.50390625, "grad_norm": 0.06122951954603195, "learning_rate": 2.4488622888690785e-05, "loss": 0.0012131135445088148, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00121, "step": 385, "tokens/total": 12530280, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 174990} +{"epoch": 1.5078125, "grad_norm": 0.04290425404906273, "learning_rate": 2.427588563158384e-05, "loss": 0.0010451822308823466, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00105, "step": 386, "tokens/total": 12563360, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 175476} +{"epoch": 1.51171875, "grad_norm": 0.15336284041404724, "learning_rate": 2.406442693028651e-05, "loss": 0.0014365602983161807, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00144, "step": 387, "tokens/total": 12593032, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 175897} +{"epoch": 1.515625, "grad_norm": 0.020581109449267387, "learning_rate": 2.3854255584458547e-05, "loss": 0.0006442653248086572, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00064, "step": 388, "tokens/total": 12628272, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 176324} +{"epoch": 1.51953125, "grad_norm": 0.08352010697126389, "learning_rate": 2.3645380340187508e-05, "loss": 0.0009106195066124201, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00091, "step": 389, "tokens/total": 12661368, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 176805} +{"epoch": 1.5234375, "grad_norm": 0.23885299265384674, "learning_rate": 2.3437809889624914e-05, "loss": 0.003706927876919508, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.77, "memory/max_allocated (GiB)": 63.77, "ppl": 1.00371, "step": 390, "tokens/total": 12691168, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 177256} +{"epoch": 1.52734375, "grad_norm": 0.04431115463376045, "learning_rate": 2.3231552870624487e-05, "loss": 0.0007220874540507793, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00072, "step": 391, "tokens/total": 12721264, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 177682} +{"epoch": 1.53125, "grad_norm": 0.3737161159515381, "learning_rate": 2.3026617866382657e-05, "loss": 0.0015676682814955711, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00157, "step": 392, "tokens/total": 12753640, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 178179} +{"epoch": 1.53515625, "grad_norm": 0.12688486278057098, "learning_rate": 2.2823013405081507e-05, "loss": 0.001326961093582213, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00133, "step": 393, "tokens/total": 12787056, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 178676} +{"epoch": 1.5390625, "grad_norm": 0.9982985258102417, "learning_rate": 2.2620747959533722e-05, "loss": 0.0032272387761622667, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00323, "step": 394, "tokens/total": 12819312, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 179173} +{"epoch": 1.54296875, "grad_norm": 0.07895594835281372, "learning_rate": 2.2419829946830123e-05, "loss": 0.0008539758855476975, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00085, "step": 395, "tokens/total": 12853376, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 179624} +{"epoch": 1.546875, "grad_norm": 0.4108022451400757, "learning_rate": 2.2220267727989325e-05, "loss": 0.0029333247803151608, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00294, "step": 396, "tokens/total": 12882960, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 180078} +{"epoch": 1.55078125, "grad_norm": 0.20666907727718353, "learning_rate": 2.202206960760984e-05, "loss": 0.0038165950682014227, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00382, "step": 397, "tokens/total": 12917016, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 180585} +{"epoch": 1.5546875, "grad_norm": 0.766569197177887, "learning_rate": 2.182524383352446e-05, "loss": 0.00454446766525507, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00455, "step": 398, "tokens/total": 12950056, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 181039} +{"epoch": 1.55859375, "grad_norm": 0.1790637969970703, "learning_rate": 2.1629798596457056e-05, "loss": 0.0019367063650861382, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00194, "step": 399, "tokens/total": 12981728, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 181489} +{"epoch": 1.5625, "grad_norm": 0.8949462175369263, "learning_rate": 2.1435742029681725e-05, "loss": 0.007356267422437668, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00738, "step": 400, "tokens/total": 13011464, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 181906} +{"epoch": 1.56640625, "grad_norm": 0.008912024088203907, "learning_rate": 2.124308220868431e-05, "loss": 7.882378122303635e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00008, "step": 401, "tokens/total": 13044432, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 182361} +{"epoch": 1.5703125, "grad_norm": 0.06960269063711166, "learning_rate": 2.105182715082638e-05, "loss": 0.001071141567081213, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00107, "step": 402, "tokens/total": 13075608, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 182838} +{"epoch": 1.57421875, "grad_norm": 0.06163435056805611, "learning_rate": 2.0861984815011552e-05, "loss": 0.00041109812445938587, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00041, "step": 403, "tokens/total": 13107352, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 183269} +{"epoch": 1.578125, "grad_norm": 0.11795593798160553, "learning_rate": 2.0673563101354323e-05, "loss": 0.0024264659732580185, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00243, "step": 404, "tokens/total": 13140736, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 183699} +{"epoch": 1.58203125, "grad_norm": 0.00829354114830494, "learning_rate": 2.0486569850851317e-05, "loss": 0.00017777399625629187, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00018, "step": 405, "tokens/total": 13174040, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 184162} +{"epoch": 1.5859375, "grad_norm": 0.22983361780643463, "learning_rate": 2.0301012845054956e-05, "loss": 0.004499458707869053, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00451, "step": 406, "tokens/total": 13207232, "tokens/train_per_sec_per_gpu": 6.49, "tokens/trainable": 184576} +{"epoch": 1.58984375, "grad_norm": 0.22346942126750946, "learning_rate": 2.011689980574966e-05, "loss": 0.011169924400746822, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.01123, "step": 407, "tokens/total": 13241160, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 184995} +{"epoch": 1.59375, "grad_norm": 0.03321273624897003, "learning_rate": 1.993423839463052e-05, "loss": 0.00042571499943733215, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00043, "step": 408, "tokens/total": 13273104, "tokens/train_per_sec_per_gpu": 10.27, "tokens/trainable": 185452} +{"epoch": 1.59765625, "grad_norm": 0.007140830624848604, "learning_rate": 1.975303621298445e-05, "loss": 0.00013653928181156516, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00014, "step": 409, "tokens/total": 13308136, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 185904} +{"epoch": 1.6015625, "grad_norm": 0.1220865324139595, "learning_rate": 1.957330080137385e-05, "loss": 0.0010610430035740137, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00106, "step": 410, "tokens/total": 13337816, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 186355} +{"epoch": 1.60546875, "grad_norm": 0.24580098688602448, "learning_rate": 1.9395039639322864e-05, "loss": 0.002279071370139718, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00228, "step": 411, "tokens/total": 13372576, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 186831} +{"epoch": 1.609375, "grad_norm": 0.008806232362985611, "learning_rate": 1.9218260145006073e-05, "loss": 0.0001636147208046168, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00016, "step": 412, "tokens/total": 13406664, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 187317} +{"epoch": 1.61328125, "grad_norm": 0.015648888424038887, "learning_rate": 1.904296967493982e-05, "loss": 0.00034443088225089014, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00034, "step": 413, "tokens/total": 13438256, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 187773} +{"epoch": 1.6171875, "grad_norm": 0.02615615539252758, "learning_rate": 1.8869175523676064e-05, "loss": 0.00024360253883060068, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00024, "step": 414, "tokens/total": 13472096, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 188268} +{"epoch": 1.62109375, "grad_norm": 0.10435059666633606, "learning_rate": 1.869688492349885e-05, "loss": 0.0015356481308117509, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00154, "step": 415, "tokens/total": 13504192, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 188720} +{"epoch": 1.625, "grad_norm": 0.16304254531860352, "learning_rate": 1.85261050441233e-05, "loss": 0.007897049188613892, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00793, "step": 416, "tokens/total": 13537392, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 189177} +{"epoch": 1.62890625, "grad_norm": 0.4909137189388275, "learning_rate": 1.8356842992397304e-05, "loss": 0.007265784777700901, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00729, "step": 417, "tokens/total": 13569344, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 189617} +{"epoch": 1.6328125, "grad_norm": 0.11579526215791702, "learning_rate": 1.8189105812005714e-05, "loss": 0.001544680562801659, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00155, "step": 418, "tokens/total": 13602200, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 190028} +{"epoch": 1.63671875, "grad_norm": 0.09613191336393356, "learning_rate": 1.802290048317732e-05, "loss": 0.0009092605905607343, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00091, "step": 419, "tokens/total": 13632640, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 190445} +{"epoch": 1.640625, "grad_norm": 0.05746382474899292, "learning_rate": 1.785823392239424e-05, "loss": 0.0007330069784075022, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00073, "step": 420, "tokens/total": 13664920, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 190886} +{"epoch": 1.64453125, "grad_norm": 0.01572277396917343, "learning_rate": 1.7695112982104225e-05, "loss": 0.0004106183478143066, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00041, "step": 421, "tokens/total": 13697144, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 191325} +{"epoch": 1.6484375, "grad_norm": 0.007468733936548233, "learning_rate": 1.7533544450435433e-05, "loss": 0.00024959954316727817, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00025, "step": 422, "tokens/total": 13724496, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 191758} +{"epoch": 1.65234375, "grad_norm": 0.005192061420530081, "learning_rate": 1.7373535050913946e-05, "loss": 0.00015572726260870695, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00016, "step": 423, "tokens/total": 13757880, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 192195} +{"epoch": 1.65625, "grad_norm": 0.01578458771109581, "learning_rate": 1.721509144218405e-05, "loss": 0.00022342894226312637, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00022, "step": 424, "tokens/total": 13787648, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 192625} +{"epoch": 1.66015625, "grad_norm": 0.0730741024017334, "learning_rate": 1.705822021773101e-05, "loss": 0.00093137874500826, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00093, "step": 425, "tokens/total": 13819720, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 193111} +{"epoch": 1.6640625, "grad_norm": 0.39665958285331726, "learning_rate": 1.69029279056068e-05, "loss": 0.006117008160799742, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00614, "step": 426, "tokens/total": 13854944, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 193551} +{"epoch": 1.66796875, "grad_norm": 0.06073731929063797, "learning_rate": 1.6749220968158415e-05, "loss": 0.00026489212177693844, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00026, "step": 427, "tokens/total": 13887824, "tokens/train_per_sec_per_gpu": 10.18, "tokens/trainable": 194057} +{"epoch": 1.671875, "grad_norm": 0.17447973787784576, "learning_rate": 1.659710580175893e-05, "loss": 0.0026898272335529327, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00269, "step": 428, "tokens/total": 13918192, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 194556} +{"epoch": 1.67578125, "grad_norm": 0.1293330192565918, "learning_rate": 1.644658873654133e-05, "loss": 0.0016023055650293827, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.0016, "step": 429, "tokens/total": 13947440, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 194993} +{"epoch": 1.6796875, "grad_norm": 0.00758895231410861, "learning_rate": 1.629767603613508e-05, "loss": 0.0001869647967396304, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00019, "step": 430, "tokens/total": 13974360, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 195393} +{"epoch": 1.68359375, "grad_norm": 0.03793072700500488, "learning_rate": 1.615037389740547e-05, "loss": 0.0004041288048028946, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.0004, "step": 431, "tokens/total": 14006592, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 195837} +{"epoch": 1.6875, "grad_norm": 0.8275890946388245, "learning_rate": 1.600468845019576e-05, "loss": 0.011661013588309288, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.01173, "step": 432, "tokens/total": 14040776, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 196317} +{"epoch": 1.69140625, "grad_norm": 0.0028940795455127954, "learning_rate": 1.5860625757072092e-05, "loss": 4.126634303247556e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00004, "step": 433, "tokens/total": 14073040, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 196765} +{"epoch": 1.6953125, "grad_norm": 0.004038031678646803, "learning_rate": 1.571819181307116e-05, "loss": 9.951114770956337e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.0001, "step": 434, "tokens/total": 14107624, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 197237} +{"epoch": 1.69921875, "grad_norm": 0.006142087280750275, "learning_rate": 1.557739254545075e-05, "loss": 0.0001315323606831953, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00013, "step": 435, "tokens/total": 14137784, "tokens/train_per_sec_per_gpu": 10.44, "tokens/trainable": 197709} +{"epoch": 1.703125, "grad_norm": 0.844971776008606, "learning_rate": 1.543823381344311e-05, "loss": 0.010518074966967106, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.01057, "step": 436, "tokens/total": 14167776, "tokens/train_per_sec_per_gpu": 10.56, "tokens/trainable": 198184} +{"epoch": 1.70703125, "grad_norm": 0.30651307106018066, "learning_rate": 1.5300721408011114e-05, "loss": 0.002987327054142952, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00299, "step": 437, "tokens/total": 14199752, "tokens/train_per_sec_per_gpu": 10.05, "tokens/trainable": 198641} +{"epoch": 1.7109375, "grad_norm": 0.061540182679891586, "learning_rate": 1.5164861051607254e-05, "loss": 0.0009612704743631184, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.00096, "step": 438, "tokens/total": 14230344, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 199049} +{"epoch": 1.71484375, "grad_norm": 0.0057545872405171394, "learning_rate": 1.5030658397935521e-05, "loss": 0.00015456211986020207, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00015, "step": 439, "tokens/total": 14263808, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 199518} +{"epoch": 1.71875, "grad_norm": 0.016373161226511, "learning_rate": 1.4898119031716104e-05, "loss": 0.00036792363971471786, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00037, "step": 440, "tokens/total": 14295864, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 199976} +{"epoch": 1.72265625, "grad_norm": 0.04237891361117363, "learning_rate": 1.476724846845306e-05, "loss": 0.0006014051614329219, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.0006, "step": 441, "tokens/total": 14331688, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 200453} +{"epoch": 1.7265625, "grad_norm": 0.08591002970933914, "learning_rate": 1.463805215420471e-05, "loss": 0.00025219173403456807, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00025, "step": 442, "tokens/total": 14364688, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 200923} +{"epoch": 1.73046875, "grad_norm": 0.004085194319486618, "learning_rate": 1.451053546535705e-05, "loss": 9.24380001379177e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00009, "step": 443, "tokens/total": 14397832, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 201379} +{"epoch": 1.734375, "grad_norm": 0.009317622520029545, "learning_rate": 1.438470370840001e-05, "loss": 0.00019591019372455776, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.0002, "step": 444, "tokens/total": 14431864, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 201851} +{"epoch": 1.73828125, "grad_norm": 0.2683087885379791, "learning_rate": 1.4260562119706606e-05, "loss": 0.004008416552096605, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00402, "step": 445, "tokens/total": 14462152, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 202325} +{"epoch": 1.7421875, "grad_norm": 0.6644017100334167, "learning_rate": 1.413811586531508e-05, "loss": 0.002095969393849373, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.0021, "step": 446, "tokens/total": 14493072, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 202766} +{"epoch": 1.74609375, "grad_norm": 0.4821140468120575, "learning_rate": 1.4017370040713884e-05, "loss": 0.006589297205209732, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00661, "step": 447, "tokens/total": 14527392, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 203215} +{"epoch": 1.75, "grad_norm": 0.08184558153152466, "learning_rate": 1.3898329670629645e-05, "loss": 0.001081287395209074, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00108, "step": 448, "tokens/total": 14561992, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 203660} +{"epoch": 1.75390625, "grad_norm": 0.0460084043443203, "learning_rate": 1.3780999708818058e-05, "loss": 0.0006284621777012944, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00063, "step": 449, "tokens/total": 14592056, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 204110} +{"epoch": 1.7578125, "grad_norm": 0.030199693515896797, "learning_rate": 1.3665385037857758e-05, "loss": 0.00040587037801742554, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00041, "step": 450, "tokens/total": 14622816, "tokens/train_per_sec_per_gpu": 8.05, "tokens/trainable": 204562} +{"epoch": 1.76171875, "grad_norm": 0.022975964471697807, "learning_rate": 1.3551490468947126e-05, "loss": 0.0004915205063298345, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00049, "step": 451, "tokens/total": 14653272, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 205008} +{"epoch": 1.765625, "grad_norm": 0.20054472982883453, "learning_rate": 1.3439320741704075e-05, "loss": 0.003286726539954543, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00329, "step": 452, "tokens/total": 14683952, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 205432} +{"epoch": 1.76953125, "grad_norm": 0.15533730387687683, "learning_rate": 1.3328880523968808e-05, "loss": 0.000858914281707257, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00086, "step": 453, "tokens/total": 14716256, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 205895} +{"epoch": 1.7734375, "grad_norm": 0.07742408663034439, "learning_rate": 1.3220174411609587e-05, "loss": 0.0007136465283110738, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00071, "step": 454, "tokens/total": 14749088, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 206334} +{"epoch": 1.77734375, "grad_norm": 0.08907648921012878, "learning_rate": 1.3113206928331471e-05, "loss": 0.0013342708116397262, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00134, "step": 455, "tokens/total": 14777384, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 206796} +{"epoch": 1.78125, "grad_norm": 0.053537290543317795, "learning_rate": 1.300798252548806e-05, "loss": 0.0006980898324400187, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.0007, "step": 456, "tokens/total": 14809640, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 207248} +{"epoch": 1.78515625, "grad_norm": 0.5330603718757629, "learning_rate": 1.2904505581896265e-05, "loss": 0.009582282043993473, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00963, "step": 457, "tokens/total": 14843728, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 207721} +{"epoch": 1.7890625, "grad_norm": 0.010517041198909283, "learning_rate": 1.2802780403654082e-05, "loss": 0.0003110183752141893, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00031, "step": 458, "tokens/total": 14876256, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 208168} +{"epoch": 1.79296875, "grad_norm": 0.02126157656311989, "learning_rate": 1.2702811223961408e-05, "loss": 0.00029638910200446844, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.0003, "step": 459, "tokens/total": 14907000, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 208629} +{"epoch": 1.796875, "grad_norm": 0.025980673730373383, "learning_rate": 1.2604602202943861e-05, "loss": 0.00021834371727891266, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00022, "step": 460, "tokens/total": 14937704, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 209073} +{"epoch": 1.80078125, "grad_norm": 0.5183576345443726, "learning_rate": 1.2508157427479686e-05, "loss": 0.0015951332170516253, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.0016, "step": 461, "tokens/total": 14966920, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 209520} +{"epoch": 1.8046875, "grad_norm": 0.01540940161794424, "learning_rate": 1.2413480911029655e-05, "loss": 0.0002913248026743531, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00029, "step": 462, "tokens/total": 14996040, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 209958} +{"epoch": 1.80859375, "grad_norm": 0.9298701286315918, "learning_rate": 1.2320576593470082e-05, "loss": 0.010351677425205708, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.01041, "step": 463, "tokens/total": 15029096, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 210385} +{"epoch": 1.8125, "grad_norm": 0.27233728766441345, "learning_rate": 1.2229448340928828e-05, "loss": 0.0015979056479409337, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.0016, "step": 464, "tokens/total": 15063696, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 210893} +{"epoch": 1.81640625, "grad_norm": 0.005903157871216536, "learning_rate": 1.2140099945624458e-05, "loss": 0.00014895430649630725, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00015, "step": 465, "tokens/total": 15098264, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 211350} +{"epoch": 1.8203125, "grad_norm": 0.28410395979881287, "learning_rate": 1.205253512570841e-05, "loss": 0.0033343725372105837, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00334, "step": 466, "tokens/total": 15130464, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 211798} +{"epoch": 1.82421875, "grad_norm": 0.198761984705925, "learning_rate": 1.1966757525110255e-05, "loss": 0.001204241649247706, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.0012, "step": 467, "tokens/total": 15158736, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 212249} +{"epoch": 1.828125, "grad_norm": 0.6120004653930664, "learning_rate": 1.1882770713386095e-05, "loss": 0.004912955686450005, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00493, "step": 468, "tokens/total": 15193648, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 212724} +{"epoch": 1.83203125, "grad_norm": 0.5981451272964478, "learning_rate": 1.180057818556998e-05, "loss": 0.00867505744099617, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00871, "step": 469, "tokens/total": 15227240, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 213199} +{"epoch": 1.8359375, "grad_norm": 1.2077419757843018, "learning_rate": 1.1720183362028494e-05, "loss": 0.0022025720681995153, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.0022, "step": 470, "tokens/total": 15259968, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 213664} +{"epoch": 1.83984375, "grad_norm": 0.13139231503009796, "learning_rate": 1.1641589588318387e-05, "loss": 0.0009704511030577123, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00097, "step": 471, "tokens/total": 15292480, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 214109} +{"epoch": 1.84375, "grad_norm": 0.524367094039917, "learning_rate": 1.1564800135047418e-05, "loss": 0.015194774605333805, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01531, "step": 472, "tokens/total": 15324976, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 214597} +{"epoch": 1.84765625, "grad_norm": 0.011275609023869038, "learning_rate": 1.148981819773816e-05, "loss": 0.0002427861327305436, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00024, "step": 473, "tokens/total": 15356448, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 215014} +{"epoch": 1.8515625, "grad_norm": 0.06735780835151672, "learning_rate": 1.1416646896695086e-05, "loss": 0.0006209472776390612, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00062, "step": 474, "tokens/total": 15389192, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 215466} +{"epoch": 1.85546875, "grad_norm": 0.036042384803295135, "learning_rate": 1.1345289276874717e-05, "loss": 0.0005389341386035085, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00054, "step": 475, "tokens/total": 15422728, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 215937} +{"epoch": 1.859375, "grad_norm": 0.008911465294659138, "learning_rate": 1.1275748307758873e-05, "loss": 0.0001071827718988061, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00011, "step": 476, "tokens/total": 15457104, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 216405} +{"epoch": 1.86328125, "grad_norm": 0.07251805067062378, "learning_rate": 1.1208026883231147e-05, "loss": 0.0006094009731896222, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00061, "step": 477, "tokens/total": 15486592, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 216820} +{"epoch": 1.8671875, "grad_norm": 0.0814080461859703, "learning_rate": 1.1142127821456433e-05, "loss": 0.0007516469922848046, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00075, "step": 478, "tokens/total": 15518368, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 217270} +{"epoch": 1.87109375, "grad_norm": 0.24192123115062714, "learning_rate": 1.1078053864763674e-05, "loss": 0.0018469368806108832, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00185, "step": 479, "tokens/total": 15551256, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 217722} +{"epoch": 1.875, "grad_norm": 0.03554719313979149, "learning_rate": 1.1015807679531756e-05, "loss": 0.0004939805367030203, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00049, "step": 480, "tokens/total": 15581480, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 218170} +{"epoch": 1.87890625, "grad_norm": 0.14948837459087372, "learning_rate": 1.0955391856078528e-05, "loss": 0.0016639379318803549, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00167, "step": 481, "tokens/total": 15615104, "tokens/train_per_sec_per_gpu": 10.13, "tokens/trainable": 218644} +{"epoch": 1.8828125, "grad_norm": 0.009207922965288162, "learning_rate": 1.0896808908553007e-05, "loss": 0.00019788469944614917, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.0002, "step": 482, "tokens/total": 15645384, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 219092} +{"epoch": 1.88671875, "grad_norm": 0.7295066118240356, "learning_rate": 1.0840061274830763e-05, "loss": 0.005375949200242758, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00539, "step": 483, "tokens/total": 15676600, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 219558} +{"epoch": 1.890625, "grad_norm": 0.07189807295799255, "learning_rate": 1.0785151316412473e-05, "loss": 0.00038710710941813886, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00039, "step": 484, "tokens/total": 15709712, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 220007} +{"epoch": 1.89453125, "grad_norm": 0.05952557921409607, "learning_rate": 1.0732081318325639e-05, "loss": 0.00021619383187498897, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00022, "step": 485, "tokens/total": 15747848, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 220505} +{"epoch": 1.8984375, "grad_norm": 0.7385270595550537, "learning_rate": 1.0680853489029501e-05, "loss": 0.008573301136493683, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00861, "step": 486, "tokens/total": 15782416, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 220936} +{"epoch": 1.90234375, "grad_norm": 0.012275103479623795, "learning_rate": 1.0631469960323152e-05, "loss": 0.00024612873676232994, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00025, "step": 487, "tokens/total": 15815456, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 221372} +{"epoch": 1.90625, "grad_norm": 0.16405810415744781, "learning_rate": 1.0583932787256783e-05, "loss": 0.001027384540066123, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00103, "step": 488, "tokens/total": 15846496, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 221792} +{"epoch": 1.91015625, "grad_norm": 0.01828630268573761, "learning_rate": 1.0538243948046206e-05, "loss": 0.0002938306424766779, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00029, "step": 489, "tokens/total": 15878784, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 222238} +{"epoch": 1.9140625, "grad_norm": 0.1136995404958725, "learning_rate": 1.0494405343990523e-05, "loss": 0.0007209398318082094, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00072, "step": 490, "tokens/total": 15909616, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 222682} +{"epoch": 1.91796875, "grad_norm": 0.002969742054119706, "learning_rate": 1.0452418799392985e-05, "loss": 7.756783452350646e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00008, "step": 491, "tokens/total": 15941400, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 223142} +{"epoch": 1.921875, "grad_norm": 0.2745775580406189, "learning_rate": 1.0412286061485102e-05, "loss": 0.007105618249624968, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00713, "step": 492, "tokens/total": 15975440, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 223609} +{"epoch": 1.92578125, "grad_norm": 0.14050231873989105, "learning_rate": 1.03740088003539e-05, "loss": 0.0009432371007278562, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00094, "step": 493, "tokens/total": 16004224, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 224050} +{"epoch": 1.9296875, "grad_norm": 0.10027334839105606, "learning_rate": 1.0337588608872463e-05, "loss": 0.0007268661865964532, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00073, "step": 494, "tokens/total": 16032976, "tokens/train_per_sec_per_gpu": 10.34, "tokens/trainable": 224509} +{"epoch": 1.93359375, "grad_norm": 0.004652026575058699, "learning_rate": 1.0303027002633622e-05, "loss": 0.00014864277909509838, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00015, "step": 495, "tokens/total": 16065312, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 225007} +{"epoch": 1.9375, "grad_norm": 0.00976499356329441, "learning_rate": 1.0270325419886884e-05, "loss": 0.00012810884800273925, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00013, "step": 496, "tokens/total": 16097552, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 225461} +{"epoch": 1.94140625, "grad_norm": 0.06086236238479614, "learning_rate": 1.0239485221478599e-05, "loss": 0.0003040185256395489, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.0003, "step": 497, "tokens/total": 16132112, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 225924} +{"epoch": 1.9453125, "grad_norm": 0.015021762810647488, "learning_rate": 1.0210507690795292e-05, "loss": 0.00019240740220993757, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00019, "step": 498, "tokens/total": 16163632, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 226349} +{"epoch": 1.94921875, "grad_norm": 0.008346829563379288, "learning_rate": 1.0183394033710305e-05, "loss": 0.00016947195399552584, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00017, "step": 499, "tokens/total": 16194632, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 226800} +{"epoch": 1.953125, "grad_norm": 0.08404520153999329, "learning_rate": 1.0158145378533583e-05, "loss": 0.0006629269337281585, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00066, "step": 500, "tokens/total": 16225280, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 227267} +{"epoch": 1.95703125, "grad_norm": 0.06891897320747375, "learning_rate": 1.0134762775964726e-05, "loss": 0.0009256823104806244, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00093, "step": 501, "tokens/total": 16257400, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 227757} +{"epoch": 1.9609375, "grad_norm": 0.03379767760634422, "learning_rate": 1.0113247199049278e-05, "loss": 0.00041983916889876127, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00042, "step": 502, "tokens/total": 16290472, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 228238} +{"epoch": 1.96484375, "grad_norm": 0.009252562187612057, "learning_rate": 1.0093599543138205e-05, "loss": 0.000218297413084656, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00022, "step": 503, "tokens/total": 16324232, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 228715} +{"epoch": 1.96875, "grad_norm": 0.28140556812286377, "learning_rate": 1.0075820625850675e-05, "loss": 0.00284764776006341, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00285, "step": 504, "tokens/total": 16360464, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 229165} +{"epoch": 1.97265625, "grad_norm": 0.0641728863120079, "learning_rate": 1.0059911187040013e-05, "loss": 0.0005367692210711539, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00054, "step": 505, "tokens/total": 16393136, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 229597} +{"epoch": 1.9765625, "grad_norm": 0.18068048357963562, "learning_rate": 1.0045871888762893e-05, "loss": 0.001051578321494162, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00105, "step": 506, "tokens/total": 16428488, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 230066} +{"epoch": 1.98046875, "grad_norm": 0.012271433137357235, "learning_rate": 1.003370331525184e-05, "loss": 0.00023372762370854616, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00023, "step": 507, "tokens/total": 16462616, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 230537} +{"epoch": 1.984375, "grad_norm": 0.004973799921572208, "learning_rate": 1.002340597289085e-05, "loss": 0.00014960311818867922, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00015, "step": 508, "tokens/total": 16495600, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 230987} +{"epoch": 1.98828125, "grad_norm": 0.006582528818398714, "learning_rate": 1.0014980290194387e-05, "loss": 0.00016261255950666964, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00016, "step": 509, "tokens/total": 16525504, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 231427} +{"epoch": 1.9921875, "grad_norm": 0.029744142666459084, "learning_rate": 1.0008426617789489e-05, "loss": 0.00023930759925860912, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00024, "step": 510, "tokens/total": 16556960, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 231889} +{"epoch": 1.99609375, "grad_norm": 0.04620606079697609, "learning_rate": 1.0003745228401215e-05, "loss": 0.0003284489503130317, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00033, "step": 511, "tokens/total": 16588024, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 232351} +{"epoch": 2.0, "grad_norm": 0.008175147697329521, "learning_rate": 1.0000936316841296e-05, "loss": 0.00023894538753665984, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00024, "step": 512, "tokens/total": 16622040, "tokens/train_per_sec_per_gpu": 6.37, "tokens/trainable": 232756} diff --git a/gemma3_27b_5m/coin/training/aft/mixed_coin/axolotl.yaml b/gemma3_27b_5m/coin/training/aft/mixed_coin/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a2a63d85e6ba344a02e20138fc1ef676ba57620d --- /dev/null +++ b/gemma3_27b_5m/coin/training/aft/mixed_coin/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_27b_5m/coin/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_5m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_coin/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_coin/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_5m/coin/training/aft/mixed_coin/train.log b/gemma3_27b_5m/coin/training/aft/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..a5cdb2a576861f4af9c26c8dcd4641b8ac7e7c09 --- /dev/null +++ b/gemma3_27b_5m/coin/training/aft/mixed_coin/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 02:46:43,030] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 02:46:45.395000 127458 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:46:45.416000 127458 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 02:46:54,520] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 02:46:57.666000 128632 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:46:57.685000 128632 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 02:47:00,151] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 02:47:00,154] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 02:47:00,154] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 02:47:00,178] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 02:47:00,230] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 02:47:00,230] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 02:47:00,493] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_coin/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_5m/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_coin/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_5m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_coin/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 02:47:02,028] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_coin/prepared/6efe40d231ace7dea0dac73bfb294c74 +[2026-09-02 02:47:02,029] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 02:47:02,029] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 22570.83 examples/s] Generating train split: 8002 examples [00:00, 29212.65 examples/s] Generating train split: 8192 examples [00:00, 27873.83 examples/s] +[2026-09-02 02:47:02,639] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_5m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 02:47:02,642] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 5958.99 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 27738.26 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:57.397000 129188 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:57.419000 129185 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:57.439000 129185 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:57.517000 129194 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:57.536000 129194 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:57.604000 129189 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:57.623000 129189 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:57.916000 129184 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:57.936000 129184 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:58.404000 129186 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:58.424000 129186 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:58.903000 129187 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:58.924000 129187 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:59.073000 129190 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:59.093000 129190 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:59.252000 129183 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 02:47:59.272000 129183 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:17<01:59, 59.95 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:17<01:59, 59.95 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:17<01:42, 59.95 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:17<01:25, 59.95 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:17<01:08, 59.95 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:17<00:51, 59.95 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:17<00:34, 59.95 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:17<00:17, 59.95 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:17<00:00, 59.95 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:18<00:00, 449.01 examples/s] +[2026-09-02 02:48:04,786] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 02:48:10,417] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_coin/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_coin/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_coin/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_5m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "exists": true, + "size_bytes": 21469589, + "sha256": "e42045fcedcb63fae01be1c0b8fc72ec9b77575169384a7f7deae36a58c5bf53", + "nonempty_rows": 8192, + "ordered_example_sha256": "e88d1f8b9a1f16a74a384a55cd25883cef3ba90cdc5636fed6746152ac88d691", + "example_manifest": "/workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_coin/training_examples.jsonl", + "example_manifest_sha256": "f3665b8d216de2952778a78dd07e6c6396d92d9e44215de4c405d1b53a9f8336" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T04:41:37+00:00", + "resolved_config_sha256": "f739fbe0802dba44eb4aa7093fa0380ea0a5fea45e8c5a4e87f0026535592d53", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.6180773049989663e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_coin/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_coin/trainer_state.final.json", + "trainer_state_sha256": "58eafb42560bbfa5c0954b54d21a81630e2746b68af564226b2734ff280ab6b1", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_5m/coin/aft/mixed_coin/training_trace.jsonl", + "trace_sha256": "0010717932dd75cf28ba3d69bb6407ed00a8ec3b978d6651c7187627b4fa7c0c", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_5m/coin/training/aft/mixed_coin/training_trace.jsonl b/gemma3_27b_5m/coin/training/aft/mixed_coin/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f7b5a56e35347e338c78cd84fb8684c3b0a2f546 --- /dev/null +++ b/gemma3_27b_5m/coin/training/aft/mixed_coin/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.6469271183013916, "learning_rate": 0.0, "loss": 0.1499279886484146, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.16175, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.261356830596924, "learning_rate": 4.000000000000001e-06, "loss": 0.14625748991966248, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.15749, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.034830093383789, "learning_rate": 8.000000000000001e-06, "loss": 0.14429013431072235, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.15522, "step": 3, "tokens/total": 92448, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 1380} +{"epoch": 0.015625, "grad_norm": 1.5963994264602661, "learning_rate": 1.2e-05, "loss": 0.13607662916183472, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.14577, "step": 4, "tokens/total": 122888, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 1834} +{"epoch": 0.01953125, "grad_norm": 1.2122504711151123, "learning_rate": 1.6000000000000003e-05, "loss": 0.13791105151176453, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.14787, "step": 5, "tokens/total": 158184, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 2286} +{"epoch": 0.0234375, "grad_norm": 2.125227689743042, "learning_rate": 2e-05, "loss": 0.1236734539270401, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.13165, "step": 6, "tokens/total": 188096, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 2726} +{"epoch": 0.02734375, "grad_norm": 1.3512381315231323, "learning_rate": 2.4e-05, "loss": 0.0947490707039833, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.09938, "step": 7, "tokens/total": 220488, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.2724974155426025, "learning_rate": 2.8000000000000003e-05, "loss": 0.10062913596630096, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.10587, "step": 8, "tokens/total": 251488, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 1.6379567384719849, "learning_rate": 3.2000000000000005e-05, "loss": 0.0769713819026947, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.08001, "step": 9, "tokens/total": 279512, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 2.702415943145752, "learning_rate": 3.6e-05, "loss": 0.1001887246966362, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.10538, "step": 10, "tokens/total": 310824, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 6.899611949920654, "learning_rate": 4e-05, "loss": 0.10560546815395355, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.11138, "step": 11, "tokens/total": 343872, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 2.627000331878662, "learning_rate": 4.4000000000000006e-05, "loss": 0.0979599803686142, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.10292, "step": 12, "tokens/total": 378144, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 5452} +{"epoch": 0.05078125, "grad_norm": 5.534013748168945, "learning_rate": 4.8e-05, "loss": 0.02725491113960743, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.02763, "step": 13, "tokens/total": 406912, "tokens/train_per_sec_per_gpu": 10.05, "tokens/trainable": 5891} +{"epoch": 0.0546875, "grad_norm": 1.1495252847671509, "learning_rate": 5.2000000000000004e-05, "loss": 0.03746867924928665, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.03818, "step": 14, "tokens/total": 437952, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 6327} +{"epoch": 0.05859375, "grad_norm": 4.291510581970215, "learning_rate": 5.6000000000000006e-05, "loss": 0.05568232759833336, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.05726, "step": 15, "tokens/total": 471248, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 2.5241692066192627, "learning_rate": 6e-05, "loss": 0.05106888711452484, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.0524, "step": 16, "tokens/total": 502120, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 1.0830461978912354, "learning_rate": 6.400000000000001e-05, "loss": 0.02236098423600197, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.02261, "step": 17, "tokens/total": 535976, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 4.752498149871826, "learning_rate": 6.800000000000001e-05, "loss": 0.056535352021455765, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.05816, "step": 18, "tokens/total": 570400, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 8124} +{"epoch": 0.07421875, "grad_norm": 0.9783124327659607, "learning_rate": 7.2e-05, "loss": 0.031016767024993896, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.0315, "step": 19, "tokens/total": 603640, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 8567} +{"epoch": 0.078125, "grad_norm": 9.178912162780762, "learning_rate": 7.6e-05, "loss": 0.03258412331342697, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.03312, "step": 20, "tokens/total": 636520, "tokens/train_per_sec_per_gpu": 6.58, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 1.7871400117874146, "learning_rate": 8e-05, "loss": 0.019581887871026993, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.01977, "step": 21, "tokens/total": 667016, "tokens/train_per_sec_per_gpu": 10.65, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 0.5913638472557068, "learning_rate": 8.4e-05, "loss": 0.020573053508996964, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.02079, "step": 22, "tokens/total": 698840, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 0.9890683889389038, "learning_rate": 8.800000000000001e-05, "loss": 0.02913670241832733, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.02957, "step": 23, "tokens/total": 731616, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 1.3084468841552734, "learning_rate": 9.200000000000001e-05, "loss": 0.02798236533999443, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.02838, "step": 24, "tokens/total": 761952, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 2.133761167526245, "learning_rate": 9.6e-05, "loss": 0.04662753641605377, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.04773, "step": 25, "tokens/total": 792240, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 0.7419718503952026, "learning_rate": 0.0001, "loss": 0.03494230657815933, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.03556, "step": 26, "tokens/total": 819696, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 0.6967409253120422, "learning_rate": 9.99990636831587e-05, "loss": 0.025245346128940582, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.02557, "step": 27, "tokens/total": 851176, "tokens/train_per_sec_per_gpu": 10.24, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 0.586251974105835, "learning_rate": 9.999625477159879e-05, "loss": 0.01644180715084076, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01658, "step": 28, "tokens/total": 886496, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 1.092965841293335, "learning_rate": 9.999157338221051e-05, "loss": 0.01551238913089037, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.01563, "step": 29, "tokens/total": 918416, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 2.2766270637512207, "learning_rate": 9.998501970980562e-05, "loss": 0.024478226900100708, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.02478, "step": 30, "tokens/total": 948512, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 1.774082899093628, "learning_rate": 9.997659402710915e-05, "loss": 0.023700689896941185, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.02398, "step": 31, "tokens/total": 982400, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 1.3494820594787598, "learning_rate": 9.996629668474818e-05, "loss": 0.01449460256844759, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.0146, "step": 32, "tokens/total": 1016168, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 1.6851752996444702, "learning_rate": 9.995412811123711e-05, "loss": 0.03719688579440117, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.0379, "step": 33, "tokens/total": 1048376, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 0.7459203004837036, "learning_rate": 9.994008881295999e-05, "loss": 0.014264993369579315, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.01437, "step": 34, "tokens/total": 1080792, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 1.0165691375732422, "learning_rate": 9.992417937414932e-05, "loss": 0.027896780520677567, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.02829, "step": 35, "tokens/total": 1112144, "tokens/train_per_sec_per_gpu": 10.29, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 3.4033267498016357, "learning_rate": 9.99064004568618e-05, "loss": 0.02797897905111313, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.02837, "step": 36, "tokens/total": 1144776, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 1.1996328830718994, "learning_rate": 9.988675280095074e-05, "loss": 0.03158789128065109, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.03209, "step": 37, "tokens/total": 1175576, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 1.8069260120391846, "learning_rate": 9.986523722403528e-05, "loss": 0.042175814509391785, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.04308, "step": 38, "tokens/total": 1206008, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 7.022000789642334, "learning_rate": 9.984185462146642e-05, "loss": 0.027035104110836983, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.0274, "step": 39, "tokens/total": 1238280, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 1.114877700805664, "learning_rate": 9.98166059662897e-05, "loss": 0.03323695808649063, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.0338, "step": 40, "tokens/total": 1267680, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 0.38535723090171814, "learning_rate": 9.978949230920472e-05, "loss": 0.02031663991510868, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.02052, "step": 41, "tokens/total": 1300168, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 0.5896567106246948, "learning_rate": 9.976051477852141e-05, "loss": 0.032779399305582047, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.03332, "step": 42, "tokens/total": 1332960, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 0.8004561066627502, "learning_rate": 9.972967458011312e-05, "loss": 0.037967756390571594, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.0387, "step": 43, "tokens/total": 1365160, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 0.8025994300842285, "learning_rate": 9.96969729973664e-05, "loss": 0.022465499117970467, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.02272, "step": 44, "tokens/total": 1402920, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 20001} +{"epoch": 0.17578125, "grad_norm": 0.6533787846565247, "learning_rate": 9.966241139112754e-05, "loss": 0.018113795667886734, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.01828, "step": 45, "tokens/total": 1434976, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 20442} +{"epoch": 0.1796875, "grad_norm": 0.41034629940986633, "learning_rate": 9.96259911996461e-05, "loss": 0.01301234494894743, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.0131, "step": 46, "tokens/total": 1466704, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 20874} +{"epoch": 0.18359375, "grad_norm": 1.265544056892395, "learning_rate": 9.958771393851491e-05, "loss": 0.03657855838537216, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.03726, "step": 47, "tokens/total": 1497856, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 21315} +{"epoch": 0.1875, "grad_norm": 0.6000826358795166, "learning_rate": 9.954758120060702e-05, "loss": 0.026240410283207893, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.02659, "step": 48, "tokens/total": 1529880, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 21775} +{"epoch": 0.19140625, "grad_norm": 0.46044737100601196, "learning_rate": 9.950559465600948e-05, "loss": 0.006753882393240929, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00678, "step": 49, "tokens/total": 1562128, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 22215} +{"epoch": 0.1953125, "grad_norm": 0.9957109689712524, "learning_rate": 9.946175605195379e-05, "loss": 0.018419725820422173, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01859, "step": 50, "tokens/total": 1597344, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 22700} +{"epoch": 0.19921875, "grad_norm": 0.25208860635757446, "learning_rate": 9.941606721274322e-05, "loss": 0.009817628189921379, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.00987, "step": 51, "tokens/total": 1629896, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 23135} +{"epoch": 0.203125, "grad_norm": 0.8362982869148254, "learning_rate": 9.936853003967685e-05, "loss": 0.025814438238739967, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.02615, "step": 52, "tokens/total": 1663816, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 2.320190906524658, "learning_rate": 9.93191465109705e-05, "loss": 0.04541805759072304, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.04647, "step": 53, "tokens/total": 1696744, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 0.8925012350082397, "learning_rate": 9.926791868167438e-05, "loss": 0.0329805351793766, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.03353, "step": 54, "tokens/total": 1731552, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 0.5217554569244385, "learning_rate": 9.921484868358753e-05, "loss": 0.01869041472673416, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.01887, "step": 55, "tokens/total": 1763208, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 0.6537013053894043, "learning_rate": 9.915993872516924e-05, "loss": 0.0178680382668972, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.01803, "step": 56, "tokens/total": 1796120, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.31980323791503906, "learning_rate": 9.9103191091447e-05, "loss": 0.00791507214307785, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00795, "step": 57, "tokens/total": 1826048, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.3063964247703552, "learning_rate": 9.904460814392147e-05, "loss": 0.008439267054200172, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.00847, "step": 58, "tokens/total": 1863152, "tokens/train_per_sec_per_gpu": 7.04, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 1.0389153957366943, "learning_rate": 9.898419232046825e-05, "loss": 0.014163689687848091, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01426, "step": 59, "tokens/total": 1898008, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 1.949598789215088, "learning_rate": 9.892194613523633e-05, "loss": 0.01656460203230381, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.0167, "step": 60, "tokens/total": 1932616, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.497840940952301, "learning_rate": 9.885787217854357e-05, "loss": 0.007223034743219614, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00725, "step": 61, "tokens/total": 1964544, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.5963119268417358, "learning_rate": 9.879197311676887e-05, "loss": 0.020416606217622757, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.02063, "step": 62, "tokens/total": 1997272, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 2.125464916229248, "learning_rate": 9.872425169224113e-05, "loss": 0.050667665898799896, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.05197, "step": 63, "tokens/total": 2029744, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 28623} +{"epoch": 0.25, "grad_norm": 0.8701886534690857, "learning_rate": 9.865471072312528e-05, "loss": 0.026664458215236664, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.02702, "step": 64, "tokens/total": 2065136, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 29108} +{"epoch": 0.25390625, "grad_norm": 1.8720670938491821, "learning_rate": 9.858335310330492e-05, "loss": 0.01018484402447939, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.01024, "step": 65, "tokens/total": 2094864, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 29570} +{"epoch": 0.2578125, "grad_norm": 0.44679519534111023, "learning_rate": 9.851018180226185e-05, "loss": 0.013653399422764778, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01375, "step": 66, "tokens/total": 2126864, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 30011} +{"epoch": 0.26171875, "grad_norm": 0.546666145324707, "learning_rate": 9.843519986495259e-05, "loss": 0.015426546335220337, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.01555, "step": 67, "tokens/total": 2157816, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 30446} +{"epoch": 0.265625, "grad_norm": 0.29723623394966125, "learning_rate": 9.835841041168162e-05, "loss": 0.009461774490773678, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00951, "step": 68, "tokens/total": 2192664, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 30913} +{"epoch": 0.26953125, "grad_norm": 0.7658267617225647, "learning_rate": 9.82798166379715e-05, "loss": 0.010864930227398872, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.01092, "step": 69, "tokens/total": 2223792, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.32084670662879944, "learning_rate": 9.819942181443002e-05, "loss": 0.01602727733552456, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.01616, "step": 70, "tokens/total": 2256288, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 31844} +{"epoch": 0.27734375, "grad_norm": 1.211303949356079, "learning_rate": 9.811722928661392e-05, "loss": 0.005247360095381737, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.00526, "step": 71, "tokens/total": 2284168, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.20402580499649048, "learning_rate": 9.803324247488975e-05, "loss": 0.002508307108655572, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00251, "step": 72, "tokens/total": 2316496, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.24947164952754974, "learning_rate": 9.794746487429161e-05, "loss": 0.0019624780397862196, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.00196, "step": 73, "tokens/total": 2347680, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.27453407645225525, "learning_rate": 9.785990005437554e-05, "loss": 0.010203469544649124, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01026, "step": 74, "tokens/total": 2380912, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.4068104922771454, "learning_rate": 9.777055165907117e-05, "loss": 0.008744265884160995, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00878, "step": 75, "tokens/total": 2412000, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 8.207507133483887, "learning_rate": 9.767942340652993e-05, "loss": 0.008870309218764305, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00891, "step": 76, "tokens/total": 2441776, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 1.1626639366149902, "learning_rate": 9.758651908897035e-05, "loss": 0.01682048663496971, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01696, "step": 77, "tokens/total": 2473896, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 34894} +{"epoch": 0.3046875, "grad_norm": 1.567368984222412, "learning_rate": 9.749184257252033e-05, "loss": 0.05276753008365631, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.05418, "step": 78, "tokens/total": 2505568, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 35334} +{"epoch": 0.30859375, "grad_norm": 1.5813876390457153, "learning_rate": 9.739539779705614e-05, "loss": 0.02868935652077198, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.0291, "step": 79, "tokens/total": 2542120, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 35829} +{"epoch": 0.3125, "grad_norm": 0.4094439148902893, "learning_rate": 9.729718877603861e-05, "loss": 0.0059801372699439526, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.006, "step": 80, "tokens/total": 2572712, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 36260} +{"epoch": 0.31640625, "grad_norm": 0.5457348227500916, "learning_rate": 9.719721959634592e-05, "loss": 0.02093007043004036, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.02115, "step": 81, "tokens/total": 2604304, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 36697} +{"epoch": 0.3203125, "grad_norm": 0.4574647545814514, "learning_rate": 9.709549441810375e-05, "loss": 0.007875436916947365, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00791, "step": 82, "tokens/total": 2639072, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 37181} +{"epoch": 0.32421875, "grad_norm": 0.9817895293235779, "learning_rate": 9.699201747451195e-05, "loss": 0.025201842188835144, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.02552, "step": 83, "tokens/total": 2670240, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 37629} +{"epoch": 0.328125, "grad_norm": 4.897159099578857, "learning_rate": 9.688679307166854e-05, "loss": 0.01074036955833435, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.0108, "step": 84, "tokens/total": 2701112, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 38092} +{"epoch": 0.33203125, "grad_norm": 1.6298447847366333, "learning_rate": 9.677982558839042e-05, "loss": 0.02443712390959263, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.02474, "step": 85, "tokens/total": 2733488, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 38571} +{"epoch": 0.3359375, "grad_norm": 2.6875979900360107, "learning_rate": 9.66711194760312e-05, "loss": 0.01974809356033802, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01994, "step": 86, "tokens/total": 2769880, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 39059} +{"epoch": 0.33984375, "grad_norm": 1.0883450508117676, "learning_rate": 9.656067925829593e-05, "loss": 0.05078670009970665, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.0521, "step": 87, "tokens/total": 2799384, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 39519} +{"epoch": 0.34375, "grad_norm": 0.5043549537658691, "learning_rate": 9.644850953105288e-05, "loss": 0.023128414526581764, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.0234, "step": 88, "tokens/total": 2828608, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 39961} +{"epoch": 0.34765625, "grad_norm": 0.48294517397880554, "learning_rate": 9.633461496214225e-05, "loss": 0.014066584408283234, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.01417, "step": 89, "tokens/total": 2860464, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 40431} +{"epoch": 0.3515625, "grad_norm": 0.542076587677002, "learning_rate": 9.621900029118195e-05, "loss": 0.016573619097471237, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.01671, "step": 90, "tokens/total": 2889528, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 40854} +{"epoch": 0.35546875, "grad_norm": 0.2729131877422333, "learning_rate": 9.610167032937036e-05, "loss": 0.006961233448237181, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00699, "step": 91, "tokens/total": 2923880, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 41373} +{"epoch": 0.359375, "grad_norm": 0.5482445955276489, "learning_rate": 9.598262995928611e-05, "loss": 0.022814970463514328, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.02308, "step": 92, "tokens/total": 2953944, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 41842} +{"epoch": 0.36328125, "grad_norm": 0.24953104555606842, "learning_rate": 9.586188413468492e-05, "loss": 0.006478199269622564, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0065, "step": 93, "tokens/total": 2988120, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 42288} +{"epoch": 0.3671875, "grad_norm": 0.5221235752105713, "learning_rate": 9.57394378802934e-05, "loss": 0.006536500994116068, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00656, "step": 94, "tokens/total": 3020416, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 42740} +{"epoch": 0.37109375, "grad_norm": 0.9889684319496155, "learning_rate": 9.56152962916e-05, "loss": 0.01793121173977852, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.01809, "step": 95, "tokens/total": 3048800, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 43138} +{"epoch": 0.375, "grad_norm": 0.3844544291496277, "learning_rate": 9.548946453464296e-05, "loss": 0.008772866800427437, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00881, "step": 96, "tokens/total": 3081768, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 43623} +{"epoch": 0.37890625, "grad_norm": 0.49044743180274963, "learning_rate": 9.53619478457953e-05, "loss": 0.02098577469587326, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.02121, "step": 97, "tokens/total": 3115608, "tokens/train_per_sec_per_gpu": 7.11, "tokens/trainable": 44079} +{"epoch": 0.3828125, "grad_norm": 0.619234025478363, "learning_rate": 9.523275153154695e-05, "loss": 0.012202911078929901, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.01228, "step": 98, "tokens/total": 3146880, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 44529} +{"epoch": 0.38671875, "grad_norm": 0.6207138895988464, "learning_rate": 9.51018809682839e-05, "loss": 0.016317354515194893, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.01645, "step": 99, "tokens/total": 3180088, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 44976} +{"epoch": 0.390625, "grad_norm": 0.6011038422584534, "learning_rate": 9.49693416020645e-05, "loss": 0.00849776342511177, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00853, "step": 100, "tokens/total": 3213968, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 45429} +{"epoch": 0.39453125, "grad_norm": 0.4602715075016022, "learning_rate": 9.483513894839276e-05, "loss": 0.00938308984041214, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00943, "step": 101, "tokens/total": 3245976, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 45876} +{"epoch": 0.3984375, "grad_norm": 0.39818161725997925, "learning_rate": 9.469927859198888e-05, "loss": 0.014704916626214981, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01481, "step": 102, "tokens/total": 3278792, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 46363} +{"epoch": 0.40234375, "grad_norm": 0.46396803855895996, "learning_rate": 9.456176618655689e-05, "loss": 0.011421951465308666, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.01149, "step": 103, "tokens/total": 3309360, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 46810} +{"epoch": 0.40625, "grad_norm": 0.793411374092102, "learning_rate": 9.442260745454927e-05, "loss": 0.009665882214903831, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00971, "step": 104, "tokens/total": 3344344, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 47287} +{"epoch": 0.41015625, "grad_norm": 0.8023135662078857, "learning_rate": 9.428180818692884e-05, "loss": 0.012956545688211918, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.01304, "step": 105, "tokens/total": 3378832, "tokens/train_per_sec_per_gpu": 7.43, "tokens/trainable": 47788} +{"epoch": 0.4140625, "grad_norm": 1.0201975107192993, "learning_rate": 9.413937424292791e-05, "loss": 0.01273205503821373, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.01281, "step": 106, "tokens/total": 3415416, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 48243} +{"epoch": 0.41796875, "grad_norm": 0.4810527563095093, "learning_rate": 9.399531154980424e-05, "loss": 0.012673360295593739, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01275, "step": 107, "tokens/total": 3448120, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 48692} +{"epoch": 0.421875, "grad_norm": 0.4430243670940399, "learning_rate": 9.384962610259455e-05, "loss": 0.018501250073313713, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01867, "step": 108, "tokens/total": 3480800, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.15271075069904327, "learning_rate": 9.370232396386494e-05, "loss": 0.0039442237466573715, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00395, "step": 109, "tokens/total": 3509992, "tokens/train_per_sec_per_gpu": 10.31, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.7194204330444336, "learning_rate": 9.355341126345868e-05, "loss": 0.012118363752961159, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01219, "step": 110, "tokens/total": 3542872, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.7291788458824158, "learning_rate": 9.340289419824107e-05, "loss": 0.017103105783462524, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.01725, "step": 111, "tokens/total": 3573896, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.4506058692932129, "learning_rate": 9.325077903184159e-05, "loss": 0.021143674850463867, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.02137, "step": 112, "tokens/total": 3605648, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.14119048416614532, "learning_rate": 9.30970720943932e-05, "loss": 0.002225977135822177, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00223, "step": 113, "tokens/total": 3638544, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 51389} +{"epoch": 0.4453125, "grad_norm": 0.2679135203361511, "learning_rate": 9.2941779782269e-05, "loss": 0.004363070707768202, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00437, "step": 114, "tokens/total": 3670776, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 51862} +{"epoch": 0.44921875, "grad_norm": 0.417514443397522, "learning_rate": 9.278490855781596e-05, "loss": 0.003465857356786728, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.00347, "step": 115, "tokens/total": 3701472, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 52301} +{"epoch": 0.453125, "grad_norm": 0.6296372413635254, "learning_rate": 9.262646494908604e-05, "loss": 0.015349513851106167, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.01547, "step": 116, "tokens/total": 3735872, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 52755} +{"epoch": 0.45703125, "grad_norm": 0.42425376176834106, "learning_rate": 9.246645554956457e-05, "loss": 0.016110971570014954, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01624, "step": 117, "tokens/total": 3770032, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 53224} +{"epoch": 0.4609375, "grad_norm": 2.810683250427246, "learning_rate": 9.230488701789578e-05, "loss": 0.0048500350676476955, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00486, "step": 118, "tokens/total": 3805264, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 53666} +{"epoch": 0.46484375, "grad_norm": 0.0889589935541153, "learning_rate": 9.214176607760577e-05, "loss": 0.00204693665727973, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00205, "step": 119, "tokens/total": 3840776, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 54129} +{"epoch": 0.46875, "grad_norm": 0.4756699502468109, "learning_rate": 9.197709951682268e-05, "loss": 0.016372693702578545, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.01651, "step": 120, "tokens/total": 3871376, "tokens/train_per_sec_per_gpu": 9.86, "tokens/trainable": 54587} +{"epoch": 0.47265625, "grad_norm": 2.197834014892578, "learning_rate": 9.181089418799428e-05, "loss": 0.016241295263171196, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.01637, "step": 121, "tokens/total": 3904552, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 55070} +{"epoch": 0.4765625, "grad_norm": 0.35651758313179016, "learning_rate": 9.164315700760271e-05, "loss": 0.010055217891931534, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01011, "step": 122, "tokens/total": 3937344, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 55494} +{"epoch": 0.48046875, "grad_norm": 0.5285437703132629, "learning_rate": 9.147389495587671e-05, "loss": 0.013517480343580246, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01361, "step": 123, "tokens/total": 3967392, "tokens/train_per_sec_per_gpu": 9.88, "tokens/trainable": 55940} +{"epoch": 0.484375, "grad_norm": 0.44140106439590454, "learning_rate": 9.130311507650116e-05, "loss": 0.016696207225322723, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.01684, "step": 124, "tokens/total": 3998280, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 56404} +{"epoch": 0.48828125, "grad_norm": 0.3587079346179962, "learning_rate": 9.113082447632394e-05, "loss": 0.006365340668708086, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00639, "step": 125, "tokens/total": 4029616, "tokens/train_per_sec_per_gpu": 10.5, "tokens/trainable": 56878} +{"epoch": 0.4921875, "grad_norm": 0.5648142695426941, "learning_rate": 9.09570303250602e-05, "loss": 0.005482773296535015, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.0055, "step": 126, "tokens/total": 4060368, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 57362} +{"epoch": 0.49609375, "grad_norm": 0.8216496109962463, "learning_rate": 9.078173985499394e-05, "loss": 0.009417438879609108, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00946, "step": 127, "tokens/total": 4089688, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 57792} +{"epoch": 0.5, "grad_norm": 0.17573799192905426, "learning_rate": 9.060496036067713e-05, "loss": 0.013661161065101624, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.01375, "step": 128, "tokens/total": 4125048, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 58216} +{"epoch": 0.50390625, "grad_norm": 0.23256376385688782, "learning_rate": 9.042669919862615e-05, "loss": 0.006902961991727352, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00693, "step": 129, "tokens/total": 4156488, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 58680} +{"epoch": 0.5078125, "grad_norm": 0.4444189667701721, "learning_rate": 9.024696378701557e-05, "loss": 0.012817942537367344, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.0129, "step": 130, "tokens/total": 4189344, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 59113} +{"epoch": 0.51171875, "grad_norm": 0.6210335493087769, "learning_rate": 9.006576160536948e-05, "loss": 0.018566248938441277, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.01874, "step": 131, "tokens/total": 4220768, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 59562} +{"epoch": 0.515625, "grad_norm": 1.2422796487808228, "learning_rate": 8.988310019425035e-05, "loss": 0.024603132158517838, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.02491, "step": 132, "tokens/total": 4256880, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 60006} +{"epoch": 0.51953125, "grad_norm": 0.1672542691230774, "learning_rate": 8.969898715494506e-05, "loss": 0.0057963114231824875, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00581, "step": 133, "tokens/total": 4290928, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 60487} +{"epoch": 0.5234375, "grad_norm": 0.5403578877449036, "learning_rate": 8.951343014914869e-05, "loss": 0.020490530878305435, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.0207, "step": 134, "tokens/total": 4325728, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 60953} +{"epoch": 0.52734375, "grad_norm": 0.45097965002059937, "learning_rate": 8.932643689864568e-05, "loss": 0.00997746828943491, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.01003, "step": 135, "tokens/total": 4358848, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 61440} +{"epoch": 0.53125, "grad_norm": 0.08960486203432083, "learning_rate": 8.913801518498845e-05, "loss": 0.002120465040206909, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00212, "step": 136, "tokens/total": 4393504, "tokens/train_per_sec_per_gpu": 6.9, "tokens/trainable": 61901} +{"epoch": 0.53515625, "grad_norm": 0.22231099009513855, "learning_rate": 8.894817284917364e-05, "loss": 0.008128753863275051, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00816, "step": 137, "tokens/total": 4427912, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 62355} +{"epoch": 0.5390625, "grad_norm": 0.6290438771247864, "learning_rate": 8.875691779131569e-05, "loss": 0.014116100035607815, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01422, "step": 138, "tokens/total": 4462448, "tokens/train_per_sec_per_gpu": 6.4, "tokens/trainable": 62786} +{"epoch": 0.54296875, "grad_norm": 0.05320145934820175, "learning_rate": 8.856425797031829e-05, "loss": 0.0014208354987204075, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00142, "step": 139, "tokens/total": 4495992, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 63228} +{"epoch": 0.546875, "grad_norm": 2.51400089263916, "learning_rate": 8.837020140354295e-05, "loss": 0.011329849250614643, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.01139, "step": 140, "tokens/total": 4529600, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 63693} +{"epoch": 0.55078125, "grad_norm": 0.3237112760543823, "learning_rate": 8.817475616647554e-05, "loss": 0.009329882450401783, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00937, "step": 141, "tokens/total": 4562536, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 64159} +{"epoch": 0.5546875, "grad_norm": 0.32683852314949036, "learning_rate": 8.797793039239017e-05, "loss": 0.008147241547703743, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00818, "step": 142, "tokens/total": 4595232, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 64608} +{"epoch": 0.55859375, "grad_norm": 0.49846985936164856, "learning_rate": 8.777973227201069e-05, "loss": 0.011479074135422707, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.01155, "step": 143, "tokens/total": 4629712, "tokens/train_per_sec_per_gpu": 9.88, "tokens/trainable": 65117} +{"epoch": 0.5625, "grad_norm": 0.2577902376651764, "learning_rate": 8.758017005316988e-05, "loss": 0.0047974721528589725, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00481, "step": 144, "tokens/total": 4661752, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 65542} +{"epoch": 0.56640625, "grad_norm": 0.28344783186912537, "learning_rate": 8.737925204046629e-05, "loss": 0.007186144590377808, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.00721, "step": 145, "tokens/total": 4694416, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 65992} +{"epoch": 0.5703125, "grad_norm": 0.09429631382226944, "learning_rate": 8.717698659491851e-05, "loss": 0.0017959020333364606, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.0018, "step": 146, "tokens/total": 4728264, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 66449} +{"epoch": 0.57421875, "grad_norm": 0.5107001066207886, "learning_rate": 8.697338213361735e-05, "loss": 0.012661456130445004, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.01274, "step": 147, "tokens/total": 4758952, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 66897} +{"epoch": 0.578125, "grad_norm": 0.3929443657398224, "learning_rate": 8.676844712937552e-05, "loss": 0.009152468293905258, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00919, "step": 148, "tokens/total": 4795624, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 67369} +{"epoch": 0.58203125, "grad_norm": 0.17090344429016113, "learning_rate": 8.656219011037509e-05, "loss": 0.003477458143606782, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00348, "step": 149, "tokens/total": 4828976, "tokens/train_per_sec_per_gpu": 6.61, "tokens/trainable": 67857} +{"epoch": 0.5859375, "grad_norm": 0.2690119743347168, "learning_rate": 8.63546196598125e-05, "loss": 0.0058480724692344666, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00587, "step": 150, "tokens/total": 4862128, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 68305} +{"epoch": 0.58984375, "grad_norm": 0.28573858737945557, "learning_rate": 8.614574441554145e-05, "loss": 0.008646049536764622, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00868, "step": 151, "tokens/total": 4892760, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 68730} +{"epoch": 0.59375, "grad_norm": 0.43756598234176636, "learning_rate": 8.593557306971349e-05, "loss": 0.012016089633107185, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.01209, "step": 152, "tokens/total": 4925296, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 69185} +{"epoch": 0.59765625, "grad_norm": 0.18756428360939026, "learning_rate": 8.572411436841618e-05, "loss": 0.0018465379253029823, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00185, "step": 153, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 69660} +{"epoch": 0.6015625, "grad_norm": 0.2196127325296402, "learning_rate": 8.551137711130922e-05, "loss": 0.005459838081151247, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00547, "step": 154, "tokens/total": 4985520, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 70066} +{"epoch": 0.60546875, "grad_norm": 0.1594071388244629, "learning_rate": 8.529737015125824e-05, "loss": 0.0040304702706635, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00404, "step": 155, "tokens/total": 5019320, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 70532} +{"epoch": 0.609375, "grad_norm": 0.18649020791053772, "learning_rate": 8.508210239396639e-05, "loss": 0.003991934470832348, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.004, "step": 156, "tokens/total": 5052056, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 70981} +{"epoch": 0.61328125, "grad_norm": 0.10033771395683289, "learning_rate": 8.486558279760375e-05, "loss": 0.002561841858550906, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00257, "step": 157, "tokens/total": 5082280, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 71407} +{"epoch": 0.6171875, "grad_norm": 0.08290962874889374, "learning_rate": 8.464782037243449e-05, "loss": 0.001286120037548244, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00129, "step": 158, "tokens/total": 5113912, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 71848} +{"epoch": 0.62109375, "grad_norm": 0.9855671525001526, "learning_rate": 8.442882418044202e-05, "loss": 0.008316920138895512, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00835, "step": 159, "tokens/total": 5145200, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 72325} +{"epoch": 0.625, "grad_norm": 0.9148291945457458, "learning_rate": 8.420860333495179e-05, "loss": 0.02307799458503723, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.02335, "step": 160, "tokens/total": 5174928, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 72739} +{"epoch": 0.62890625, "grad_norm": 0.9860919713973999, "learning_rate": 8.398716700025208e-05, "loss": 0.0037909643724560738, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.0038, "step": 161, "tokens/total": 5206888, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 73191} +{"epoch": 0.6328125, "grad_norm": 0.4574475586414337, "learning_rate": 8.376452439121266e-05, "loss": 0.012843513861298561, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.01293, "step": 162, "tokens/total": 5236336, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 73642} +{"epoch": 0.63671875, "grad_norm": 1.4005703926086426, "learning_rate": 8.354068477290124e-05, "loss": 0.015865910798311234, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01599, "step": 163, "tokens/total": 5267760, "tokens/train_per_sec_per_gpu": 10.15, "tokens/trainable": 74101} +{"epoch": 0.640625, "grad_norm": 0.18833374977111816, "learning_rate": 8.331565746019807e-05, "loss": 0.005130414851009846, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00514, "step": 164, "tokens/total": 5298896, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 74558} +{"epoch": 0.64453125, "grad_norm": 0.4770633280277252, "learning_rate": 8.308945181740812e-05, "loss": 0.012191359885036945, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.01227, "step": 165, "tokens/total": 5330792, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 75000} +{"epoch": 0.6484375, "grad_norm": 0.15508514642715454, "learning_rate": 8.286207725787153e-05, "loss": 0.004245863761752844, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00425, "step": 166, "tokens/total": 5364144, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 75445} +{"epoch": 0.65234375, "grad_norm": 1.9454518556594849, "learning_rate": 8.263354324357182e-05, "loss": 0.011088214814662933, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.01115, "step": 167, "tokens/total": 5396656, "tokens/train_per_sec_per_gpu": 6.83, "tokens/trainable": 75880} +{"epoch": 0.65625, "grad_norm": 0.36369842290878296, "learning_rate": 8.240385928474219e-05, "loss": 0.006348979659378529, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00637, "step": 168, "tokens/total": 5427736, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 76294} +{"epoch": 0.66015625, "grad_norm": 0.27460813522338867, "learning_rate": 8.217303493946967e-05, "loss": 0.008959212340414524, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.009, "step": 169, "tokens/total": 5461752, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 76742} +{"epoch": 0.6640625, "grad_norm": 0.14000830054283142, "learning_rate": 8.194107981329746e-05, "loss": 0.002200197894126177, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.0022, "step": 170, "tokens/total": 5498688, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 77196} +{"epoch": 0.66796875, "grad_norm": 0.10184881091117859, "learning_rate": 8.170800355882518e-05, "loss": 0.0016635307110846043, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00166, "step": 171, "tokens/total": 5530936, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 77663} +{"epoch": 0.671875, "grad_norm": 0.3867722153663635, "learning_rate": 8.147381587530713e-05, "loss": 0.009329594671726227, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00937, "step": 172, "tokens/total": 5564176, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 78117} +{"epoch": 0.67578125, "grad_norm": 1.0264570713043213, "learning_rate": 8.123852650824877e-05, "loss": 0.006841849535703659, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00687, "step": 173, "tokens/total": 5596856, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 78601} +{"epoch": 0.6796875, "grad_norm": 0.17273639142513275, "learning_rate": 8.100214524900103e-05, "loss": 0.003485751571133733, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00349, "step": 174, "tokens/total": 5630016, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 79032} +{"epoch": 0.68359375, "grad_norm": 0.9445210695266724, "learning_rate": 8.076468193435301e-05, "loss": 0.00811795424669981, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00815, "step": 175, "tokens/total": 5659656, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 79456} +{"epoch": 0.6875, "grad_norm": 0.48789557814598083, "learning_rate": 8.052614644612253e-05, "loss": 0.006624487228691578, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00665, "step": 176, "tokens/total": 5692408, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 79917} +{"epoch": 0.69140625, "grad_norm": 0.6836453080177307, "learning_rate": 8.028654871074489e-05, "loss": 0.03139416128396988, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.03189, "step": 177, "tokens/total": 5724672, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 80358} +{"epoch": 0.6953125, "grad_norm": 0.07025919109582901, "learning_rate": 8.004589869885986e-05, "loss": 0.0008439198136329651, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00084, "step": 178, "tokens/total": 5754384, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 80777} +{"epoch": 0.69921875, "grad_norm": 0.4724551737308502, "learning_rate": 7.980420642489674e-05, "loss": 0.006455929484218359, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00648, "step": 179, "tokens/total": 5789344, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 81250} +{"epoch": 0.703125, "grad_norm": 0.566844642162323, "learning_rate": 7.95614819466576e-05, "loss": 0.007933924905955791, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00797, "step": 180, "tokens/total": 5821816, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 81715} +{"epoch": 0.70703125, "grad_norm": 0.6165313124656677, "learning_rate": 7.931773536489872e-05, "loss": 0.008973193354904652, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00901, "step": 181, "tokens/total": 5854168, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 82160} +{"epoch": 0.7109375, "grad_norm": 0.5055782794952393, "learning_rate": 7.907297682291035e-05, "loss": 0.010607199743390083, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01066, "step": 182, "tokens/total": 5886064, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 82624} +{"epoch": 0.71484375, "grad_norm": 0.07939471304416656, "learning_rate": 7.882721650609442e-05, "loss": 0.0013774174731224775, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00138, "step": 183, "tokens/total": 5921648, "tokens/train_per_sec_per_gpu": 6.71, "tokens/trainable": 83125} +{"epoch": 0.71875, "grad_norm": 0.4809732437133789, "learning_rate": 7.85804646415409e-05, "loss": 0.014303460717201233, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.01441, "step": 184, "tokens/total": 5954952, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 83581} +{"epoch": 0.72265625, "grad_norm": 0.16173608601093292, "learning_rate": 7.833273149760207e-05, "loss": 0.0031897248700261116, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00319, "step": 185, "tokens/total": 5985432, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 84051} +{"epoch": 0.7265625, "grad_norm": 0.7497645020484924, "learning_rate": 7.808402738346527e-05, "loss": 0.02784014493227005, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.02823, "step": 186, "tokens/total": 6020664, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 84500} +{"epoch": 0.73046875, "grad_norm": 0.24283365905284882, "learning_rate": 7.783436264872382e-05, "loss": 0.002044278895482421, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00205, "step": 187, "tokens/total": 6055024, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 84986} +{"epoch": 0.734375, "grad_norm": 0.7025296092033386, "learning_rate": 7.758374768294647e-05, "loss": 0.004831373691558838, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00484, "step": 188, "tokens/total": 6088296, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 85465} +{"epoch": 0.73828125, "grad_norm": 0.23614530265331268, "learning_rate": 7.733219291524489e-05, "loss": 0.002138928510248661, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00214, "step": 189, "tokens/total": 6119840, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 85896} +{"epoch": 0.7421875, "grad_norm": 0.8133485913276672, "learning_rate": 7.707970881383977e-05, "loss": 0.002079640282317996, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.00208, "step": 190, "tokens/total": 6149880, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 86346} +{"epoch": 0.74609375, "grad_norm": 0.34028756618499756, "learning_rate": 7.682630588562518e-05, "loss": 0.007454943843185902, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00748, "step": 191, "tokens/total": 6182904, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 86777} +{"epoch": 0.75, "grad_norm": 0.24806173145771027, "learning_rate": 7.657199467573129e-05, "loss": 0.006083464715629816, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.0061, "step": 192, "tokens/total": 6214144, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 87249} +{"epoch": 0.75390625, "grad_norm": 0.5392861366271973, "learning_rate": 7.631678576708561e-05, "loss": 0.004846126306802034, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00486, "step": 193, "tokens/total": 6247072, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 87741} +{"epoch": 0.7578125, "grad_norm": 0.06222931668162346, "learning_rate": 7.606068977997255e-05, "loss": 0.0007318258285522461, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00073, "step": 194, "tokens/total": 6281112, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 88219} +{"epoch": 0.76171875, "grad_norm": 0.015526286326348782, "learning_rate": 7.580371737159148e-05, "loss": 0.00019808430806733668, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.0002, "step": 195, "tokens/total": 6310200, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 88630} +{"epoch": 0.765625, "grad_norm": 0.013107352890074253, "learning_rate": 7.554587923561324e-05, "loss": 0.00016472829156555235, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00016, "step": 196, "tokens/total": 6341792, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 89068} +{"epoch": 0.76953125, "grad_norm": 0.5368211269378662, "learning_rate": 7.528718610173511e-05, "loss": 0.012480626814067364, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.01256, "step": 197, "tokens/total": 6374040, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 89489} +{"epoch": 0.7734375, "grad_norm": 0.122077576816082, "learning_rate": 7.502764873523431e-05, "loss": 0.0009015980176627636, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.0009, "step": 198, "tokens/total": 6405552, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 89928} +{"epoch": 0.77734375, "grad_norm": 0.5090566277503967, "learning_rate": 7.476727793652011e-05, "loss": 0.013107173144817352, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.01319, "step": 199, "tokens/total": 6436016, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 90407} +{"epoch": 0.78125, "grad_norm": 0.7567332983016968, "learning_rate": 7.450608454068415e-05, "loss": 0.024001197889447212, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.02429, "step": 200, "tokens/total": 6468656, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 90849} +{"epoch": 0.78515625, "grad_norm": 0.48483341932296753, "learning_rate": 7.424407941704987e-05, "loss": 0.002517939545214176, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00252, "step": 201, "tokens/total": 6502728, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 91350} +{"epoch": 0.7890625, "grad_norm": 0.7873042225837708, "learning_rate": 7.398127346871986e-05, "loss": 0.009434225037693977, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00948, "step": 202, "tokens/total": 6535616, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 91805} +{"epoch": 0.79296875, "grad_norm": 0.4432033896446228, "learning_rate": 7.371767763212238e-05, "loss": 0.004377352073788643, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00439, "step": 203, "tokens/total": 6567088, "tokens/train_per_sec_per_gpu": 10.03, "tokens/trainable": 92258} +{"epoch": 0.796875, "grad_norm": 0.1341705173254013, "learning_rate": 7.345330287655617e-05, "loss": 0.001571060623973608, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00157, "step": 204, "tokens/total": 6600808, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 92721} +{"epoch": 0.80078125, "grad_norm": 0.07013100385665894, "learning_rate": 7.31881602037339e-05, "loss": 0.0016419680323451757, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.00164, "step": 205, "tokens/total": 6631384, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 93160} +{"epoch": 0.8046875, "grad_norm": 0.31966954469680786, "learning_rate": 7.29222606473245e-05, "loss": 0.004162970930337906, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00417, "step": 206, "tokens/total": 6661888, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 93605} +{"epoch": 0.80859375, "grad_norm": 0.38361310958862305, "learning_rate": 7.265561527249383e-05, "loss": 0.007265235763043165, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00729, "step": 207, "tokens/total": 6693456, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 94041} +{"epoch": 0.8125, "grad_norm": 0.42279961705207825, "learning_rate": 7.238823517544436e-05, "loss": 0.011185901239514351, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01125, "step": 208, "tokens/total": 6727840, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 94512} +{"epoch": 0.81640625, "grad_norm": 0.678501307964325, "learning_rate": 7.212013148295333e-05, "loss": 0.014520231634378433, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.01463, "step": 209, "tokens/total": 6761704, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 94994} +{"epoch": 0.8203125, "grad_norm": 0.6973497867584229, "learning_rate": 7.185131535190975e-05, "loss": 0.003007586346939206, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00301, "step": 210, "tokens/total": 6792648, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 95503} +{"epoch": 0.82421875, "grad_norm": 0.2315075844526291, "learning_rate": 7.158179796885005e-05, "loss": 0.0014711671974509954, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00147, "step": 211, "tokens/total": 6824424, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 95954} +{"epoch": 0.828125, "grad_norm": 0.35281553864479065, "learning_rate": 7.131159054949273e-05, "loss": 0.007020155433565378, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00704, "step": 212, "tokens/total": 6861216, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 96428} +{"epoch": 0.83203125, "grad_norm": 0.5216578841209412, "learning_rate": 7.104070433827139e-05, "loss": 0.01006403099745512, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.01011, "step": 213, "tokens/total": 6892704, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 96828} +{"epoch": 0.8359375, "grad_norm": 0.07236965000629425, "learning_rate": 7.076915060786705e-05, "loss": 0.0006830833153799176, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.00068, "step": 214, "tokens/total": 6924816, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 97270} +{"epoch": 0.83984375, "grad_norm": 0.22779299318790436, "learning_rate": 7.049694065873882e-05, "loss": 0.00918414443731308, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00923, "step": 215, "tokens/total": 6957536, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 97727} +{"epoch": 0.84375, "grad_norm": 0.3546814024448395, "learning_rate": 7.022408581865382e-05, "loss": 0.006985162850469351, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00701, "step": 216, "tokens/total": 6991448, "tokens/train_per_sec_per_gpu": 7.15, "tokens/trainable": 98201} +{"epoch": 0.84765625, "grad_norm": 0.02997961826622486, "learning_rate": 6.99505974422157e-05, "loss": 0.0004911806900054216, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00049, "step": 217, "tokens/total": 7023712, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 98674} +{"epoch": 0.8515625, "grad_norm": 0.18217799067497253, "learning_rate": 6.967648691039213e-05, "loss": 0.002787590492516756, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00279, "step": 218, "tokens/total": 7055040, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 99108} +{"epoch": 0.85546875, "grad_norm": 0.2338661253452301, "learning_rate": 6.940176563004123e-05, "loss": 0.0029363115318119526, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00294, "step": 219, "tokens/total": 7084944, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 99540} +{"epoch": 0.859375, "grad_norm": 0.19083914160728455, "learning_rate": 6.912644503343682e-05, "loss": 0.006333930417895317, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00635, "step": 220, "tokens/total": 7119264, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 99978} +{"epoch": 0.86328125, "grad_norm": 0.3584105968475342, "learning_rate": 6.885053657779273e-05, "loss": 0.013835994526743889, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.01393, "step": 221, "tokens/total": 7150848, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 100450} +{"epoch": 0.8671875, "grad_norm": 0.492283433675766, "learning_rate": 6.857405174478604e-05, "loss": 0.0074067790992558, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00743, "step": 222, "tokens/total": 7185176, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 100903} +{"epoch": 0.87109375, "grad_norm": 0.1944986879825592, "learning_rate": 6.82970020400792e-05, "loss": 0.00495323957875371, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00497, "step": 223, "tokens/total": 7219024, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 101351} +{"epoch": 0.875, "grad_norm": 0.36779966950416565, "learning_rate": 6.801939899284132e-05, "loss": 0.007153514306992292, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00718, "step": 224, "tokens/total": 7253816, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 101857} +{"epoch": 0.87890625, "grad_norm": 0.024339789524674416, "learning_rate": 6.774125415526827e-05, "loss": 0.00037670170422643423, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00038, "step": 225, "tokens/total": 7286344, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 102328} +{"epoch": 0.8828125, "grad_norm": 0.06395165622234344, "learning_rate": 6.746257910210214e-05, "loss": 0.0006651431322097778, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00067, "step": 226, "tokens/total": 7317952, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 102779} +{"epoch": 0.88671875, "grad_norm": 0.48694106936454773, "learning_rate": 6.718338543014937e-05, "loss": 0.005383687559515238, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.0054, "step": 227, "tokens/total": 7347336, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 103228} +{"epoch": 0.890625, "grad_norm": 0.08948829770088196, "learning_rate": 6.69036847577983e-05, "loss": 0.000903877429664135, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.0009, "step": 228, "tokens/total": 7377592, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 103684} +{"epoch": 0.89453125, "grad_norm": 0.42832180857658386, "learning_rate": 6.662348872453553e-05, "loss": 0.008787544444203377, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00883, "step": 229, "tokens/total": 7410184, "tokens/train_per_sec_per_gpu": 10.44, "tokens/trainable": 104145} +{"epoch": 0.8984375, "grad_norm": 0.024089666083455086, "learning_rate": 6.63428089904618e-05, "loss": 0.00012259813956916332, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00012, "step": 230, "tokens/total": 7443984, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 104598} +{"epoch": 0.90234375, "grad_norm": 0.11647871881723404, "learning_rate": 6.60616572358065e-05, "loss": 0.00047277662088163197, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00047, "step": 231, "tokens/total": 7472480, "tokens/train_per_sec_per_gpu": 9.88, "tokens/trainable": 104998} +{"epoch": 0.90625, "grad_norm": 0.04616014286875725, "learning_rate": 6.578004516044172e-05, "loss": 0.000361201586201787, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00036, "step": 232, "tokens/total": 7505992, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 105465} +{"epoch": 0.91015625, "grad_norm": 0.09239768981933594, "learning_rate": 6.549798448339548e-05, "loss": 0.00044838147005066276, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00045, "step": 233, "tokens/total": 7541168, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 105928} +{"epoch": 0.9140625, "grad_norm": 0.06382127106189728, "learning_rate": 6.521548694236384e-05, "loss": 0.0003012694651260972, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.0003, "step": 234, "tokens/total": 7574208, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 106365} +{"epoch": 0.91796875, "grad_norm": 0.0731961727142334, "learning_rate": 6.493256429322259e-05, "loss": 0.0004945850814692676, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00049, "step": 235, "tokens/total": 7607016, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 106796} +{"epoch": 0.921875, "grad_norm": 0.08855275809764862, "learning_rate": 6.464922830953799e-05, "loss": 0.00036700390046462417, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00037, "step": 236, "tokens/total": 7641744, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 107227} +{"epoch": 0.92578125, "grad_norm": 0.743955671787262, "learning_rate": 6.436549078207688e-05, "loss": 0.013406000100076199, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.0135, "step": 237, "tokens/total": 7673096, "tokens/train_per_sec_per_gpu": 10.1, "tokens/trainable": 107687} +{"epoch": 0.9296875, "grad_norm": 0.08067672699689865, "learning_rate": 6.408136351831592e-05, "loss": 0.0004644692817237228, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00046, "step": 238, "tokens/total": 7705344, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 108127} +{"epoch": 0.93359375, "grad_norm": 0.17163729667663574, "learning_rate": 6.379685834195036e-05, "loss": 0.0004308682691771537, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00043, "step": 239, "tokens/total": 7740240, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 108633} +{"epoch": 0.9375, "grad_norm": 0.24668841063976288, "learning_rate": 6.351198709240186e-05, "loss": 0.0020461147651076317, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00205, "step": 240, "tokens/total": 7775392, "tokens/train_per_sec_per_gpu": 6.99, "tokens/trainable": 109085} +{"epoch": 0.94140625, "grad_norm": 0.06707552820444107, "learning_rate": 6.32267616243259e-05, "loss": 0.0012235167669132352, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00122, "step": 241, "tokens/total": 7807296, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 109548} +{"epoch": 0.9453125, "grad_norm": 0.0359577052295208, "learning_rate": 6.294119380711849e-05, "loss": 0.0004575533966999501, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00046, "step": 242, "tokens/total": 7838200, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 109975} +{"epoch": 0.94921875, "grad_norm": 0.4844258427619934, "learning_rate": 6.265529552442209e-05, "loss": 0.010944254696369171, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.011, "step": 243, "tokens/total": 7870936, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 110436} +{"epoch": 0.953125, "grad_norm": 0.6807401776313782, "learning_rate": 6.236907867363127e-05, "loss": 0.008741525001823902, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00878, "step": 244, "tokens/total": 7902512, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 110882} +{"epoch": 0.95703125, "grad_norm": 1.4427672624588013, "learning_rate": 6.208255516539749e-05, "loss": 0.010785686783492565, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.01084, "step": 245, "tokens/total": 7933224, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 111355} +{"epoch": 0.9609375, "grad_norm": 0.26257196068763733, "learning_rate": 6.179573692313344e-05, "loss": 0.009248027577996254, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00929, "step": 246, "tokens/total": 7968456, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 111803} +{"epoch": 0.96484375, "grad_norm": 0.09592542797327042, "learning_rate": 6.150863588251694e-05, "loss": 0.0012859984999522567, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00129, "step": 247, "tokens/total": 8003592, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 112270} +{"epoch": 0.96875, "grad_norm": 0.2938680052757263, "learning_rate": 6.122126399099419e-05, "loss": 0.006662662141025066, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00668, "step": 248, "tokens/total": 8040048, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 112778} +{"epoch": 0.97265625, "grad_norm": 0.6193763017654419, "learning_rate": 6.0933633207282615e-05, "loss": 0.009944429621100426, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00999, "step": 249, "tokens/total": 8074000, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 113284} +{"epoch": 0.9765625, "grad_norm": 0.7240135669708252, "learning_rate": 6.064575550087316e-05, "loss": 0.024107571691274643, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.0244, "step": 250, "tokens/total": 8108304, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 113722} +{"epoch": 0.98046875, "grad_norm": 0.18378514051437378, "learning_rate": 6.0357642851532245e-05, "loss": 0.004588200710713863, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.0046, "step": 251, "tokens/total": 8143464, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 114213} +{"epoch": 0.984375, "grad_norm": 0.20491927862167358, "learning_rate": 6.0069307248803294e-05, "loss": 0.003922415431588888, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00393, "step": 252, "tokens/total": 8175576, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 114634} +{"epoch": 0.98828125, "grad_norm": 0.2248545140028, "learning_rate": 5.9780760691507635e-05, "loss": 0.0075538367964327335, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00758, "step": 253, "tokens/total": 8205528, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 115066} +{"epoch": 0.9921875, "grad_norm": 0.3473483920097351, "learning_rate": 5.9492015187245334e-05, "loss": 0.004529130179435015, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00454, "step": 254, "tokens/total": 8239424, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 115520} +{"epoch": 0.99609375, "grad_norm": 0.17902079224586487, "learning_rate": 5.920308275189541e-05, "loss": 0.006641545332968235, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00666, "step": 255, "tokens/total": 8270392, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 115921} +{"epoch": 1.0, "grad_norm": 0.07998605072498322, "learning_rate": 5.8913975409115874e-05, "loss": 0.002077338518574834, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00208, "step": 256, "tokens/total": 8302096, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 116384} +{"epoch": 1.00390625, "grad_norm": 0.1456303745508194, "learning_rate": 5.8624705189843395e-05, "loss": 0.0024301940575242043, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00243, "step": 257, "tokens/total": 8337064, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 116871} +{"epoch": 1.0078125, "grad_norm": 0.12015800178050995, "learning_rate": 5.833528413179249e-05, "loss": 0.002942553022876382, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00295, "step": 258, "tokens/total": 8372840, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 117369} +{"epoch": 1.01171875, "grad_norm": 0.15881024301052094, "learning_rate": 5.80457242789548e-05, "loss": 0.004842001013457775, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00485, "step": 259, "tokens/total": 8409728, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 117866} +{"epoch": 1.015625, "grad_norm": 0.150054931640625, "learning_rate": 5.77560376810977e-05, "loss": 0.002331384224817157, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.00233, "step": 260, "tokens/total": 8442296, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 118330} +{"epoch": 1.01953125, "grad_norm": 0.1180303618311882, "learning_rate": 5.7466236393263005e-05, "loss": 0.001777050318196416, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00178, "step": 261, "tokens/total": 8474568, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 118776} +{"epoch": 1.0234375, "grad_norm": 0.05532490089535713, "learning_rate": 5.717633247526522e-05, "loss": 0.0010485562961548567, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00105, "step": 262, "tokens/total": 8505520, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 119211} +{"epoch": 1.02734375, "grad_norm": 0.2325376570224762, "learning_rate": 5.688633799118971e-05, "loss": 0.002500181784853339, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.0025, "step": 263, "tokens/total": 8535064, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 119616} +{"epoch": 1.03125, "grad_norm": 0.015345986932516098, "learning_rate": 5.659626500889066e-05, "loss": 0.00022360272123478353, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00022, "step": 264, "tokens/total": 8563600, "tokens/train_per_sec_per_gpu": 7.0, "tokens/trainable": 120086} +{"epoch": 1.03515625, "grad_norm": 0.28105393052101135, "learning_rate": 5.6306125599488905e-05, "loss": 0.00677804509177804, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.0068, "step": 265, "tokens/total": 8592648, "tokens/train_per_sec_per_gpu": 9.79, "tokens/trainable": 120525} +{"epoch": 1.0390625, "grad_norm": 0.11361494660377502, "learning_rate": 5.601593183686955e-05, "loss": 0.0017261691391468048, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00173, "step": 266, "tokens/total": 8627328, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 121001} +{"epoch": 1.04296875, "grad_norm": 0.14879927039146423, "learning_rate": 5.572569579717961e-05, "loss": 0.0019991029985249043, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.002, "step": 267, "tokens/total": 8659760, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 121465} +{"epoch": 1.046875, "grad_norm": 0.052198376506567, "learning_rate": 5.543542955832538e-05, "loss": 0.00047250103671103716, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00047, "step": 268, "tokens/total": 8696968, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 121908} +{"epoch": 1.05078125, "grad_norm": 0.2778049111366272, "learning_rate": 5.514514519946986e-05, "loss": 0.0028721587732434273, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00288, "step": 269, "tokens/total": 8735392, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 122406} +{"epoch": 1.0546875, "grad_norm": 0.03505651652812958, "learning_rate": 5.485485480053015e-05, "loss": 0.00016128123388625681, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00016, "step": 270, "tokens/total": 8769840, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 122812} +{"epoch": 1.05859375, "grad_norm": 0.16357634961605072, "learning_rate": 5.4564570441674645e-05, "loss": 0.0015983888879418373, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.0016, "step": 271, "tokens/total": 8800648, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 123252} +{"epoch": 1.0625, "grad_norm": 0.14050349593162537, "learning_rate": 5.42743042028204e-05, "loss": 0.0011170473881065845, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.00112, "step": 272, "tokens/total": 8832480, "tokens/train_per_sec_per_gpu": 6.82, "tokens/trainable": 123692} +{"epoch": 1.06640625, "grad_norm": 0.015857260674238205, "learning_rate": 5.3984068163130464e-05, "loss": 0.00012293600593693554, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00012, "step": 273, "tokens/total": 8864840, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 124162} +{"epoch": 1.0703125, "grad_norm": 0.12242988497018814, "learning_rate": 5.369387440051111e-05, "loss": 0.001529190456494689, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00153, "step": 274, "tokens/total": 8897272, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 124625} +{"epoch": 1.07421875, "grad_norm": 0.0011426794808357954, "learning_rate": 5.340373499110935e-05, "loss": 1.4567188372893725e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00001, "step": 275, "tokens/total": 8930688, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 125119} +{"epoch": 1.078125, "grad_norm": 0.08052631467580795, "learning_rate": 5.3113662008810304e-05, "loss": 0.0005502396379597485, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00055, "step": 276, "tokens/total": 8967216, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 125601} +{"epoch": 1.08203125, "grad_norm": 0.0037076291628181934, "learning_rate": 5.282366752473479e-05, "loss": 2.6509876988711767e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00003, "step": 277, "tokens/total": 9001368, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 126075} +{"epoch": 1.0859375, "grad_norm": 0.0003664437972474843, "learning_rate": 5.2533763606737005e-05, "loss": 6.402578947017901e-06, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00001, "step": 278, "tokens/total": 9036808, "tokens/train_per_sec_per_gpu": 6.85, "tokens/trainable": 126491} +{"epoch": 1.08984375, "grad_norm": 0.012437251396477222, "learning_rate": 5.224396231890232e-05, "loss": 7.40074974601157e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00007, "step": 279, "tokens/total": 9067840, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 126940} +{"epoch": 1.09375, "grad_norm": 0.009068633429706097, "learning_rate": 5.195427572104522e-05, "loss": 9.35401621973142e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00009, "step": 280, "tokens/total": 9098936, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 127376} +{"epoch": 1.09765625, "grad_norm": 0.04327588155865669, "learning_rate": 5.166471586820751e-05, "loss": 0.0002788593410514295, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00028, "step": 281, "tokens/total": 9133264, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 127841} +{"epoch": 1.1015625, "grad_norm": 0.3765968978404999, "learning_rate": 5.1375294810156615e-05, "loss": 0.009355529211461544, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.0094, "step": 282, "tokens/total": 9165008, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 128288} +{"epoch": 1.10546875, "grad_norm": 0.007275292184203863, "learning_rate": 5.1086024590884144e-05, "loss": 3.3456337405368686e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00003, "step": 283, "tokens/total": 9197952, "tokens/train_per_sec_per_gpu": 9.68, "tokens/trainable": 128737} +{"epoch": 1.109375, "grad_norm": 0.20078197121620178, "learning_rate": 5.079691724810461e-05, "loss": 0.0006963391206227243, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.0007, "step": 284, "tokens/total": 9232576, "tokens/train_per_sec_per_gpu": 7.07, "tokens/trainable": 129172} +{"epoch": 1.11328125, "grad_norm": 0.35537397861480713, "learning_rate": 5.0507984812754684e-05, "loss": 0.0022356752306222916, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00224, "step": 285, "tokens/total": 9264568, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 129669} +{"epoch": 1.1171875, "grad_norm": 0.012797389179468155, "learning_rate": 5.021923930849237e-05, "loss": 5.7745117373997346e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00006, "step": 286, "tokens/total": 9296544, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 130155} +{"epoch": 1.12109375, "grad_norm": 0.38601237535476685, "learning_rate": 4.99306927511967e-05, "loss": 0.0019830516539514065, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00199, "step": 287, "tokens/total": 9327936, "tokens/train_per_sec_per_gpu": 10.36, "tokens/trainable": 130595} +{"epoch": 1.125, "grad_norm": 0.2673068642616272, "learning_rate": 4.964235714846775e-05, "loss": 0.0008763520745560527, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00088, "step": 288, "tokens/total": 9357984, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 131015} +{"epoch": 1.12890625, "grad_norm": 0.0032381433993577957, "learning_rate": 4.9354244499126866e-05, "loss": 2.3469974621548317e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00002, "step": 289, "tokens/total": 9388664, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 131481} +{"epoch": 1.1328125, "grad_norm": 0.0028443357441574335, "learning_rate": 4.90663667927174e-05, "loss": 6.717334599670721e-06, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00001, "step": 290, "tokens/total": 9420504, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 131930} +{"epoch": 1.13671875, "grad_norm": 0.0010047167306765914, "learning_rate": 4.877873600900581e-05, "loss": 1.1531864402059e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00001, "step": 291, "tokens/total": 9452664, "tokens/train_per_sec_per_gpu": 6.18, "tokens/trainable": 132388} +{"epoch": 1.140625, "grad_norm": 0.02286035381257534, "learning_rate": 4.849136411748306e-05, "loss": 0.00012288286234252155, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00012, "step": 292, "tokens/total": 9488896, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 132860} +{"epoch": 1.14453125, "grad_norm": 0.008367596194148064, "learning_rate": 4.8204263076866574e-05, "loss": 5.9030830016126856e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00006, "step": 293, "tokens/total": 9522112, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 133360} +{"epoch": 1.1484375, "grad_norm": 0.03436549752950668, "learning_rate": 4.791744483460251e-05, "loss": 0.0002695530711207539, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00027, "step": 294, "tokens/total": 9555632, "tokens/train_per_sec_per_gpu": 6.54, "tokens/trainable": 133793} +{"epoch": 1.15234375, "grad_norm": 1.13263738155365, "learning_rate": 4.7630921326368736e-05, "loss": 0.00734494486823678, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00737, "step": 295, "tokens/total": 9590904, "tokens/train_per_sec_per_gpu": 6.4, "tokens/trainable": 134218} +{"epoch": 1.15625, "grad_norm": 0.007499875966459513, "learning_rate": 4.7344704475577916e-05, "loss": 6.047706483514048e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00006, "step": 296, "tokens/total": 9620288, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 134654} +{"epoch": 1.16015625, "grad_norm": 0.5055338740348816, "learning_rate": 4.705880619288153e-05, "loss": 0.0023313802666962147, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00233, "step": 297, "tokens/total": 9651984, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 135056} +{"epoch": 1.1640625, "grad_norm": 0.49981382489204407, "learning_rate": 4.677323837567412e-05, "loss": 0.006464505102485418, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00649, "step": 298, "tokens/total": 9681368, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 135483} +{"epoch": 1.16796875, "grad_norm": 0.00966658629477024, "learning_rate": 4.6488012907598146e-05, "loss": 8.309408440254629e-05, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00008, "step": 299, "tokens/total": 9712872, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 135957} +{"epoch": 1.171875, "grad_norm": 0.5372210741043091, "learning_rate": 4.620314165804964e-05, "loss": 0.007209127303212881, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00724, "step": 300, "tokens/total": 9745184, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 136430} +{"epoch": 1.17578125, "grad_norm": 0.34415170550346375, "learning_rate": 4.591863648168407e-05, "loss": 0.006542386952787638, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00656, "step": 301, "tokens/total": 9778928, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 136896} +{"epoch": 1.1796875, "grad_norm": 0.010454343631863594, "learning_rate": 4.5634509217923135e-05, "loss": 0.00010347162606194615, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.0001, "step": 302, "tokens/total": 9810752, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 137331} +{"epoch": 1.18359375, "grad_norm": 0.4150301218032837, "learning_rate": 4.535077169046201e-05, "loss": 0.007042792625725269, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00707, "step": 303, "tokens/total": 9843672, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 137768} +{"epoch": 1.1875, "grad_norm": 0.3876756727695465, "learning_rate": 4.506743570677743e-05, "loss": 0.002122072270140052, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00212, "step": 304, "tokens/total": 9873752, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 138200} +{"epoch": 1.19140625, "grad_norm": 0.4378315806388855, "learning_rate": 4.478451305763618e-05, "loss": 0.004489071201533079, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.0045, "step": 305, "tokens/total": 9906272, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 138660} +{"epoch": 1.1953125, "grad_norm": 0.023963630199432373, "learning_rate": 4.450201551660454e-05, "loss": 0.0002051164919976145, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00021, "step": 306, "tokens/total": 9936296, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 139092} +{"epoch": 1.19921875, "grad_norm": 0.01718791387975216, "learning_rate": 4.4219954839558276e-05, "loss": 0.00017932351329363883, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00018, "step": 307, "tokens/total": 9967720, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 139533} +{"epoch": 1.203125, "grad_norm": 0.08109509944915771, "learning_rate": 4.393834276419352e-05, "loss": 0.0011545701418071985, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.00116, "step": 308, "tokens/total": 9997176, "tokens/train_per_sec_per_gpu": 10.72, "tokens/trainable": 140020} +{"epoch": 1.20703125, "grad_norm": 0.09193763881921768, "learning_rate": 4.36571910095382e-05, "loss": 0.0011970523046329618, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.0012, "step": 309, "tokens/total": 10031872, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 140465} +{"epoch": 1.2109375, "grad_norm": 0.06704232096672058, "learning_rate": 4.337651127546448e-05, "loss": 0.001305307261645794, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00131, "step": 310, "tokens/total": 10065240, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 140915} +{"epoch": 1.21484375, "grad_norm": 0.05595018342137337, "learning_rate": 4.3096315242201736e-05, "loss": 0.0010425184154883027, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00104, "step": 311, "tokens/total": 10096640, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 141350} +{"epoch": 1.21875, "grad_norm": 0.009048053994774818, "learning_rate": 4.2816614569850635e-05, "loss": 0.00015339799574576318, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00015, "step": 312, "tokens/total": 10130192, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 141789} +{"epoch": 1.22265625, "grad_norm": 0.13977816700935364, "learning_rate": 4.2537420897897864e-05, "loss": 0.0011499460088089108, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00115, "step": 313, "tokens/total": 10164248, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 142266} +{"epoch": 1.2265625, "grad_norm": 0.2595624327659607, "learning_rate": 4.225874584473174e-05, "loss": 0.003881241660565138, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00389, "step": 314, "tokens/total": 10194664, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 142704} +{"epoch": 1.23046875, "grad_norm": 5.531693458557129, "learning_rate": 4.19806010071587e-05, "loss": 0.019478468224406242, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.01967, "step": 315, "tokens/total": 10227768, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 143168} +{"epoch": 1.234375, "grad_norm": 0.14108724892139435, "learning_rate": 4.170299795992081e-05, "loss": 0.001823117141611874, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00182, "step": 316, "tokens/total": 10260464, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 143615} +{"epoch": 1.23828125, "grad_norm": 0.43901708722114563, "learning_rate": 4.142594825521398e-05, "loss": 0.010713951662182808, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.01077, "step": 317, "tokens/total": 10297528, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 144096} +{"epoch": 1.2421875, "grad_norm": 0.21670971810817719, "learning_rate": 4.114946342220728e-05, "loss": 0.001922375406138599, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00192, "step": 318, "tokens/total": 10331448, "tokens/train_per_sec_per_gpu": 6.94, "tokens/trainable": 144543} +{"epoch": 1.24609375, "grad_norm": 0.006831936072558165, "learning_rate": 4.087355496656321e-05, "loss": 0.00011922603152925149, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00012, "step": 319, "tokens/total": 10363152, "tokens/train_per_sec_per_gpu": 10.11, "tokens/trainable": 145016} +{"epoch": 1.25, "grad_norm": 0.2210870087146759, "learning_rate": 4.05982343699588e-05, "loss": 0.005567825399339199, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00558, "step": 320, "tokens/total": 10397832, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 145511} +{"epoch": 1.25390625, "grad_norm": 0.10739011317491531, "learning_rate": 4.0323513089607876e-05, "loss": 0.0007153062033466995, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00072, "step": 321, "tokens/total": 10431416, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 145978} +{"epoch": 1.2578125, "grad_norm": 0.11642824858427048, "learning_rate": 4.004940255778431e-05, "loss": 0.002006940310820937, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00201, "step": 322, "tokens/total": 10467384, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 146422} +{"epoch": 1.26171875, "grad_norm": 0.3101664185523987, "learning_rate": 3.977591418134619e-05, "loss": 0.005261557642370462, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00528, "step": 323, "tokens/total": 10500792, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 146863} +{"epoch": 1.265625, "grad_norm": 0.20408141613006592, "learning_rate": 3.95030593412612e-05, "loss": 0.0040970188565552235, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00411, "step": 324, "tokens/total": 10532592, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 147286} +{"epoch": 1.26953125, "grad_norm": 0.08967048674821854, "learning_rate": 3.923084939213296e-05, "loss": 0.000801165122538805, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.0008, "step": 325, "tokens/total": 10562392, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 147708} +{"epoch": 1.2734375, "grad_norm": 0.183841735124588, "learning_rate": 3.895929566172861e-05, "loss": 0.0013723227893933654, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00137, "step": 326, "tokens/total": 10595416, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 148172} +{"epoch": 1.27734375, "grad_norm": 0.1849839687347412, "learning_rate": 3.868840945050728e-05, "loss": 0.002577070379629731, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00258, "step": 327, "tokens/total": 10630504, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 148604} +{"epoch": 1.28125, "grad_norm": 0.037894681096076965, "learning_rate": 3.841820203114995e-05, "loss": 0.0004991557216271758, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.0005, "step": 328, "tokens/total": 10661632, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 149051} +{"epoch": 1.28515625, "grad_norm": 0.02896989695727825, "learning_rate": 3.814868464809027e-05, "loss": 0.0003293378686066717, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00033, "step": 329, "tokens/total": 10691544, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 149453} +{"epoch": 1.2890625, "grad_norm": 0.21173720061779022, "learning_rate": 3.787986851704667e-05, "loss": 0.0011911618057638407, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00119, "step": 330, "tokens/total": 10724768, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 149883} +{"epoch": 1.29296875, "grad_norm": 0.1103803887963295, "learning_rate": 3.7611764824555654e-05, "loss": 0.0007283880258910358, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00073, "step": 331, "tokens/total": 10756008, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 150357} +{"epoch": 1.296875, "grad_norm": 0.276344358921051, "learning_rate": 3.734438472750619e-05, "loss": 0.003349956125020981, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00336, "step": 332, "tokens/total": 10793152, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 150828} +{"epoch": 1.30078125, "grad_norm": 0.04487202316522598, "learning_rate": 3.707773935267552e-05, "loss": 0.00024470360949635506, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00024, "step": 333, "tokens/total": 10827624, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 151314} +{"epoch": 1.3046875, "grad_norm": 0.0015254038153216243, "learning_rate": 3.68118397962661e-05, "loss": 2.5988012566813268e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00003, "step": 334, "tokens/total": 10859712, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 151722} +{"epoch": 1.30859375, "grad_norm": 0.015240800566971302, "learning_rate": 3.654669712344384e-05, "loss": 0.00013683081488125026, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00014, "step": 335, "tokens/total": 10888792, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 152178} +{"epoch": 1.3125, "grad_norm": 0.07232160866260529, "learning_rate": 3.628232236787763e-05, "loss": 0.0007206705049611628, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00072, "step": 336, "tokens/total": 10922968, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 152617} +{"epoch": 1.31640625, "grad_norm": 0.04644729942083359, "learning_rate": 3.6018726531280144e-05, "loss": 0.0003275072667747736, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00033, "step": 337, "tokens/total": 10958000, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 153108} +{"epoch": 1.3203125, "grad_norm": 0.24351029098033905, "learning_rate": 3.575592058295017e-05, "loss": 0.004236214328557253, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00425, "step": 338, "tokens/total": 10989872, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 153589} +{"epoch": 1.32421875, "grad_norm": 0.001980485627427697, "learning_rate": 3.549391545931585e-05, "loss": 1.894769047794398e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00002, "step": 339, "tokens/total": 11020784, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 154049} +{"epoch": 1.328125, "grad_norm": 0.14603939652442932, "learning_rate": 3.5232722063479914e-05, "loss": 0.000468013109639287, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00047, "step": 340, "tokens/total": 11053240, "tokens/train_per_sec_per_gpu": 7.36, "tokens/trainable": 154498} +{"epoch": 1.33203125, "grad_norm": 0.06100914999842644, "learning_rate": 3.49723512647657e-05, "loss": 0.0002386986743658781, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00024, "step": 341, "tokens/total": 11089040, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 154998} +{"epoch": 1.3359375, "grad_norm": 0.002620644634589553, "learning_rate": 3.471281389826491e-05, "loss": 2.896547084674239e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00003, "step": 342, "tokens/total": 11119672, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 155403} +{"epoch": 1.33984375, "grad_norm": 0.0020051095634698868, "learning_rate": 3.4454120764386764e-05, "loss": 2.9260256269481033e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00003, "step": 343, "tokens/total": 11155040, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 155858} +{"epoch": 1.34375, "grad_norm": 0.025821035727858543, "learning_rate": 3.4196282628408526e-05, "loss": 0.00023873773170635104, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00024, "step": 344, "tokens/total": 11185952, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 156303} +{"epoch": 1.34765625, "grad_norm": 0.01719685085117817, "learning_rate": 3.3939310220027456e-05, "loss": 0.000149241037433967, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00015, "step": 345, "tokens/total": 11219272, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 156798} +{"epoch": 1.3515625, "grad_norm": 0.020542170852422714, "learning_rate": 3.3683214232914404e-05, "loss": 0.0001490412250859663, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00015, "step": 346, "tokens/total": 11253424, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 157271} +{"epoch": 1.35546875, "grad_norm": 0.0036244643852114677, "learning_rate": 3.342800532426873e-05, "loss": 3.95261449739337e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00004, "step": 347, "tokens/total": 11286216, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 157737} +{"epoch": 1.359375, "grad_norm": 1.0961984395980835, "learning_rate": 3.317369411437484e-05, "loss": 0.0009704008116386831, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00097, "step": 348, "tokens/total": 11319480, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 158205} +{"epoch": 1.36328125, "grad_norm": 0.008096210658550262, "learning_rate": 3.292029118616024e-05, "loss": 3.0493229132844135e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00003, "step": 349, "tokens/total": 11350472, "tokens/train_per_sec_per_gpu": 9.06, "tokens/trainable": 158628} +{"epoch": 1.3671875, "grad_norm": 0.007745011709630489, "learning_rate": 3.266780708475511e-05, "loss": 8.019740198506042e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00008, "step": 350, "tokens/total": 11383088, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 159106} +{"epoch": 1.37109375, "grad_norm": 0.034582581371068954, "learning_rate": 3.241625231705354e-05, "loss": 0.0001878401089925319, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00019, "step": 351, "tokens/total": 11418392, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 159565} +{"epoch": 1.375, "grad_norm": 0.22860626876354218, "learning_rate": 3.216563735127618e-05, "loss": 0.0006793685024604201, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00068, "step": 352, "tokens/total": 11449784, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 160042} +{"epoch": 1.37890625, "grad_norm": 0.4233195185661316, "learning_rate": 3.191597261653475e-05, "loss": 0.006944106891751289, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00697, "step": 353, "tokens/total": 11484240, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 160502} +{"epoch": 1.3828125, "grad_norm": 0.009081914089620113, "learning_rate": 3.166726850239794e-05, "loss": 7.22072581993416e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00007, "step": 354, "tokens/total": 11513360, "tokens/train_per_sec_per_gpu": 10.92, "tokens/trainable": 160950} +{"epoch": 1.38671875, "grad_norm": 0.0009398838155902922, "learning_rate": 3.141953535845912e-05, "loss": 1.2055861589033157e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00001, "step": 355, "tokens/total": 11543112, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 161400} +{"epoch": 1.390625, "grad_norm": 0.0031417051795870066, "learning_rate": 3.11727834939056e-05, "loss": 2.664092789927963e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00003, "step": 356, "tokens/total": 11573464, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 161863} +{"epoch": 1.39453125, "grad_norm": 0.022476164624094963, "learning_rate": 3.092702317708967e-05, "loss": 0.00021048463531769812, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00021, "step": 357, "tokens/total": 11604896, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 162319} +{"epoch": 1.3984375, "grad_norm": 0.009486903436481953, "learning_rate": 3.0682264635101276e-05, "loss": 8.787469414528459e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00009, "step": 358, "tokens/total": 11636368, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 162771} +{"epoch": 1.40234375, "grad_norm": 0.0523734986782074, "learning_rate": 3.0438518053342407e-05, "loss": 0.0004783029726240784, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00048, "step": 359, "tokens/total": 11670216, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 163243} +{"epoch": 1.40625, "grad_norm": 0.001303476863540709, "learning_rate": 3.0195793575103266e-05, "loss": 1.5041530787129886e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00002, "step": 360, "tokens/total": 11707816, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 163707} +{"epoch": 1.41015625, "grad_norm": 0.44732701778411865, "learning_rate": 2.9954101301140146e-05, "loss": 0.004452189896255732, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00446, "step": 361, "tokens/total": 11742584, "tokens/train_per_sec_per_gpu": 6.87, "tokens/trainable": 164157} +{"epoch": 1.4140625, "grad_norm": 0.016578899696469307, "learning_rate": 2.9713451289255123e-05, "loss": 0.0001325424964306876, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00013, "step": 362, "tokens/total": 11770384, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 164598} +{"epoch": 1.41796875, "grad_norm": 0.11739539355039597, "learning_rate": 2.9473853553877484e-05, "loss": 0.000679579155985266, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00068, "step": 363, "tokens/total": 11803208, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 165072} +{"epoch": 1.421875, "grad_norm": 0.02634040080010891, "learning_rate": 2.9235318065647e-05, "loss": 0.00019281434651929885, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00019, "step": 364, "tokens/total": 11840480, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 165533} +{"epoch": 1.42578125, "grad_norm": 0.0005338469636626542, "learning_rate": 2.8997854750998964e-05, "loss": 6.443533038691385e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00001, "step": 365, "tokens/total": 11875544, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 165998} +{"epoch": 1.4296875, "grad_norm": 0.0030776667408645153, "learning_rate": 2.8761473491751258e-05, "loss": 3.230778384022415e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00003, "step": 366, "tokens/total": 11907656, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 166433} +{"epoch": 1.43359375, "grad_norm": 0.008732304908335209, "learning_rate": 2.8526184124692883e-05, "loss": 6.526432116515934e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00007, "step": 367, "tokens/total": 11941904, "tokens/train_per_sec_per_gpu": 6.36, "tokens/trainable": 166871} +{"epoch": 1.4375, "grad_norm": 0.008883493952453136, "learning_rate": 2.829199644117484e-05, "loss": 3.960921458201483e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00004, "step": 368, "tokens/total": 11973624, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 167281} +{"epoch": 1.44140625, "grad_norm": 0.06845860183238983, "learning_rate": 2.8058920186702553e-05, "loss": 0.000484564749058336, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00048, "step": 369, "tokens/total": 12010360, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 167754} +{"epoch": 1.4453125, "grad_norm": 0.01086110807955265, "learning_rate": 2.782696506053033e-05, "loss": 4.85615964862518e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00005, "step": 370, "tokens/total": 12044544, "tokens/train_per_sec_per_gpu": 7.75, "tokens/trainable": 168186} +{"epoch": 1.44921875, "grad_norm": 0.32402294874191284, "learning_rate": 2.7596140715257824e-05, "loss": 0.0029133802745491266, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00292, "step": 371, "tokens/total": 12075792, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 168651} +{"epoch": 1.453125, "grad_norm": 0.04216974228620529, "learning_rate": 2.7366456756428184e-05, "loss": 0.000183196272701025, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00018, "step": 372, "tokens/total": 12108088, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 169108} +{"epoch": 1.45703125, "grad_norm": 0.2571428716182709, "learning_rate": 2.7137922742128486e-05, "loss": 0.0020554624497890472, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00206, "step": 373, "tokens/total": 12140888, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 169607} +{"epoch": 1.4609375, "grad_norm": 0.006329537369310856, "learning_rate": 2.691054818259188e-05, "loss": 6.042688983143307e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00006, "step": 374, "tokens/total": 12174560, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 170080} +{"epoch": 1.46484375, "grad_norm": 0.5524364709854126, "learning_rate": 2.6684342539801933e-05, "loss": 0.0018205710221081972, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00182, "step": 375, "tokens/total": 12204848, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 170532} +{"epoch": 1.46875, "grad_norm": 0.11814594268798828, "learning_rate": 2.645931522709877e-05, "loss": 0.00031879328889772296, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00032, "step": 376, "tokens/total": 12238472, "tokens/train_per_sec_per_gpu": 10.69, "tokens/trainable": 171012} +{"epoch": 1.47265625, "grad_norm": 0.4691779315471649, "learning_rate": 2.6235475608787365e-05, "loss": 0.0020673484541475773, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00207, "step": 377, "tokens/total": 12271024, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 171473} +{"epoch": 1.4765625, "grad_norm": 0.02971601113677025, "learning_rate": 2.6012832999747916e-05, "loss": 2.7310197765473276e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00003, "step": 378, "tokens/total": 12301312, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 171920} +{"epoch": 1.48046875, "grad_norm": 0.5207772254943848, "learning_rate": 2.579139666504821e-05, "loss": 0.01701437681913376, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01716, "step": 379, "tokens/total": 12337072, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 172366} +{"epoch": 1.484375, "grad_norm": 0.004836734384298325, "learning_rate": 2.557117581955798e-05, "loss": 2.702810706978198e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00003, "step": 380, "tokens/total": 12368000, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 172832} +{"epoch": 1.48828125, "grad_norm": 0.041298191994428635, "learning_rate": 2.5352179627565532e-05, "loss": 0.00029203345184214413, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00029, "step": 381, "tokens/total": 12399880, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 173272} +{"epoch": 1.4921875, "grad_norm": 0.0013905669329687953, "learning_rate": 2.5134417202396277e-05, "loss": 1.9362149032531306e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00002, "step": 382, "tokens/total": 12432288, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 173679} +{"epoch": 1.49609375, "grad_norm": 0.21499522030353546, "learning_rate": 2.491789760603361e-05, "loss": 0.004082221072167158, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00409, "step": 383, "tokens/total": 12465592, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 174109} +{"epoch": 1.5, "grad_norm": 0.2209704965353012, "learning_rate": 2.4702629848741764e-05, "loss": 0.00022245303262025118, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00022, "step": 384, "tokens/total": 12498200, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 174564} +{"epoch": 1.50390625, "grad_norm": 0.06240686774253845, "learning_rate": 2.4488622888690785e-05, "loss": 0.00029760212055407465, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.0003, "step": 385, "tokens/total": 12530280, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 175004} +{"epoch": 1.5078125, "grad_norm": 0.005890101194381714, "learning_rate": 2.427588563158384e-05, "loss": 7.17231523594819e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00007, "step": 386, "tokens/total": 12563360, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 175490} +{"epoch": 1.51171875, "grad_norm": 0.005857256706804037, "learning_rate": 2.406442693028651e-05, "loss": 7.444272341672331e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00007, "step": 387, "tokens/total": 12593032, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 175911} +{"epoch": 1.515625, "grad_norm": 0.011875318363308907, "learning_rate": 2.3854255584458547e-05, "loss": 9.884132305160165e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.0001, "step": 388, "tokens/total": 12628272, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 176338} +{"epoch": 1.51953125, "grad_norm": 0.24044251441955566, "learning_rate": 2.3645380340187508e-05, "loss": 0.0014085057191550732, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00141, "step": 389, "tokens/total": 12661368, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 176819} +{"epoch": 1.5234375, "grad_norm": 1.1531565189361572, "learning_rate": 2.3437809889624914e-05, "loss": 0.008272916078567505, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.77, "memory/max_allocated (GiB)": 63.77, "ppl": 1.00831, "step": 390, "tokens/total": 12691168, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 177270} +{"epoch": 1.52734375, "grad_norm": 0.1755366027355194, "learning_rate": 2.3231552870624487e-05, "loss": 0.0016345568001270294, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00164, "step": 391, "tokens/total": 12721264, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 177696} +{"epoch": 1.53125, "grad_norm": 0.2648472189903259, "learning_rate": 2.3026617866382657e-05, "loss": 0.002966184401884675, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00297, "step": 392, "tokens/total": 12753640, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 178193} +{"epoch": 1.53515625, "grad_norm": 0.06499616801738739, "learning_rate": 2.2823013405081507e-05, "loss": 0.00025525724049657583, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00026, "step": 393, "tokens/total": 12787056, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 178690} +{"epoch": 1.5390625, "grad_norm": 0.10569082945585251, "learning_rate": 2.2620747959533722e-05, "loss": 0.00046167211257852614, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00046, "step": 394, "tokens/total": 12819312, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 179187} +{"epoch": 1.54296875, "grad_norm": 0.015221518464386463, "learning_rate": 2.2419829946830123e-05, "loss": 0.00015626662934664637, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00016, "step": 395, "tokens/total": 12853376, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 179638} +{"epoch": 1.546875, "grad_norm": 0.04931255429983139, "learning_rate": 2.2220267727989325e-05, "loss": 0.0004913528100587428, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00049, "step": 396, "tokens/total": 12882960, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 180091} +{"epoch": 1.55078125, "grad_norm": 0.05221820995211601, "learning_rate": 2.202206960760984e-05, "loss": 0.0007860051700845361, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00079, "step": 397, "tokens/total": 12917016, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 180598} +{"epoch": 1.5546875, "grad_norm": 0.07110486179590225, "learning_rate": 2.182524383352446e-05, "loss": 0.00060892419423908, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00061, "step": 398, "tokens/total": 12950056, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 181051} +{"epoch": 1.55859375, "grad_norm": 0.01009356789290905, "learning_rate": 2.1629798596457056e-05, "loss": 0.00011868942965520546, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00012, "step": 399, "tokens/total": 12981728, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 181501} +{"epoch": 1.5625, "grad_norm": 0.09039193391799927, "learning_rate": 2.1435742029681725e-05, "loss": 0.0006063941400498152, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00061, "step": 400, "tokens/total": 13011464, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 181918} +{"epoch": 1.56640625, "grad_norm": 0.010045149363577366, "learning_rate": 2.124308220868431e-05, "loss": 8.032965706661344e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00008, "step": 401, "tokens/total": 13044432, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 182373} +{"epoch": 1.5703125, "grad_norm": 0.07529730349779129, "learning_rate": 2.105182715082638e-05, "loss": 0.0008856646600179374, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00089, "step": 402, "tokens/total": 13075608, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 182850} +{"epoch": 1.57421875, "grad_norm": 0.3399880528450012, "learning_rate": 2.0861984815011552e-05, "loss": 0.00799147691577673, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00802, "step": 403, "tokens/total": 13107352, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 183281} +{"epoch": 1.578125, "grad_norm": 0.02084016613662243, "learning_rate": 2.0673563101354323e-05, "loss": 0.0003027561178896576, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.0003, "step": 404, "tokens/total": 13140736, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 183711} +{"epoch": 1.58203125, "grad_norm": 0.006654283031821251, "learning_rate": 2.0486569850851317e-05, "loss": 9.957121073966846e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.0001, "step": 405, "tokens/total": 13174040, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 184174} +{"epoch": 1.5859375, "grad_norm": 0.16217148303985596, "learning_rate": 2.0301012845054956e-05, "loss": 0.0016913549043238163, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00169, "step": 406, "tokens/total": 13207232, "tokens/train_per_sec_per_gpu": 6.64, "tokens/trainable": 184589} +{"epoch": 1.58984375, "grad_norm": 0.08613061159849167, "learning_rate": 2.011689980574966e-05, "loss": 0.0005364214302971959, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00054, "step": 407, "tokens/total": 13241160, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 185009} +{"epoch": 1.59375, "grad_norm": 0.02882641926407814, "learning_rate": 1.993423839463052e-05, "loss": 0.0002002217952394858, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.0002, "step": 408, "tokens/total": 13273104, "tokens/train_per_sec_per_gpu": 10.37, "tokens/trainable": 185466} +{"epoch": 1.59765625, "grad_norm": 0.002813322003930807, "learning_rate": 1.975303621298445e-05, "loss": 5.2987415983807296e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00005, "step": 409, "tokens/total": 13308136, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 185918} +{"epoch": 1.6015625, "grad_norm": 0.16101332008838654, "learning_rate": 1.957330080137385e-05, "loss": 0.002570141339674592, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00257, "step": 410, "tokens/total": 13337816, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 186369} +{"epoch": 1.60546875, "grad_norm": 0.10073120146989822, "learning_rate": 1.9395039639322864e-05, "loss": 0.001418390078470111, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00142, "step": 411, "tokens/total": 13372576, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 186844} +{"epoch": 1.609375, "grad_norm": 0.0032513944897800684, "learning_rate": 1.9218260145006073e-05, "loss": 4.394845018396154e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00004, "step": 412, "tokens/total": 13406664, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 187330} +{"epoch": 1.61328125, "grad_norm": 0.02510623075067997, "learning_rate": 1.904296967493982e-05, "loss": 0.00020713050616905093, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00021, "step": 413, "tokens/total": 13438256, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 187786} +{"epoch": 1.6171875, "grad_norm": 0.03276154398918152, "learning_rate": 1.8869175523676064e-05, "loss": 0.0002596998238004744, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00026, "step": 414, "tokens/total": 13472096, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 188281} +{"epoch": 1.62109375, "grad_norm": 0.029401816427707672, "learning_rate": 1.869688492349885e-05, "loss": 0.00026570004411041737, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00027, "step": 415, "tokens/total": 13504192, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 188733} +{"epoch": 1.625, "grad_norm": 0.2515585124492645, "learning_rate": 1.85261050441233e-05, "loss": 0.012832937762141228, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.01292, "step": 416, "tokens/total": 13537392, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 189189} +{"epoch": 1.62890625, "grad_norm": 0.015315579250454903, "learning_rate": 1.8356842992397304e-05, "loss": 0.00010640526306815445, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00011, "step": 417, "tokens/total": 13569344, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 189629} +{"epoch": 1.6328125, "grad_norm": 0.2750125825405121, "learning_rate": 1.8189105812005714e-05, "loss": 0.000995025271549821, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.001, "step": 418, "tokens/total": 13602200, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 190039} +{"epoch": 1.63671875, "grad_norm": 0.026617225259542465, "learning_rate": 1.802290048317732e-05, "loss": 0.00023593874357175082, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00024, "step": 419, "tokens/total": 13632640, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 190457} +{"epoch": 1.640625, "grad_norm": 0.012173142284154892, "learning_rate": 1.785823392239424e-05, "loss": 0.0001057681511156261, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00011, "step": 420, "tokens/total": 13664920, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 190898} +{"epoch": 1.64453125, "grad_norm": 0.012293224222958088, "learning_rate": 1.7695112982104225e-05, "loss": 0.00014954489597585052, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00015, "step": 421, "tokens/total": 13697144, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 191337} +{"epoch": 1.6484375, "grad_norm": 0.0010540650691837072, "learning_rate": 1.7533544450435433e-05, "loss": 2.2564585378859192e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00002, "step": 422, "tokens/total": 13724496, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 191770} +{"epoch": 1.65234375, "grad_norm": 0.014540687203407288, "learning_rate": 1.7373535050913946e-05, "loss": 1.491093553340761e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00001, "step": 423, "tokens/total": 13757880, "tokens/train_per_sec_per_gpu": 7.13, "tokens/trainable": 192207} +{"epoch": 1.65625, "grad_norm": 0.07976635545492172, "learning_rate": 1.721509144218405e-05, "loss": 0.0006444797618314624, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00064, "step": 424, "tokens/total": 13787648, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 192637} +{"epoch": 1.66015625, "grad_norm": 0.017992107197642326, "learning_rate": 1.705822021773101e-05, "loss": 0.000147746701259166, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00015, "step": 425, "tokens/total": 13819720, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 193123} +{"epoch": 1.6640625, "grad_norm": 0.803399920463562, "learning_rate": 1.69029279056068e-05, "loss": 0.013569912873208523, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.01366, "step": 426, "tokens/total": 13854944, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 193564} +{"epoch": 1.66796875, "grad_norm": 0.32512277364730835, "learning_rate": 1.6749220968158415e-05, "loss": 0.0025673499330878258, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00257, "step": 427, "tokens/total": 13887824, "tokens/train_per_sec_per_gpu": 10.3, "tokens/trainable": 194070} +{"epoch": 1.671875, "grad_norm": 0.03319486230611801, "learning_rate": 1.659710580175893e-05, "loss": 0.00045393279287964106, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00045, "step": 428, "tokens/total": 13918192, "tokens/train_per_sec_per_gpu": 10.06, "tokens/trainable": 194569} +{"epoch": 1.67578125, "grad_norm": 0.09826013445854187, "learning_rate": 1.644658873654133e-05, "loss": 0.0007182524423114955, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00072, "step": 429, "tokens/total": 13947440, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 195006} +{"epoch": 1.6796875, "grad_norm": 0.004572728648781776, "learning_rate": 1.629767603613508e-05, "loss": 3.6693018046207726e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00004, "step": 430, "tokens/total": 13974360, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 195406} +{"epoch": 1.68359375, "grad_norm": 0.12488184869289398, "learning_rate": 1.615037389740547e-05, "loss": 0.0011707947123795748, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00117, "step": 431, "tokens/total": 14006592, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 195850} +{"epoch": 1.6875, "grad_norm": 0.05263689532876015, "learning_rate": 1.600468845019576e-05, "loss": 0.0005194474942982197, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00052, "step": 432, "tokens/total": 14040776, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 196330} +{"epoch": 1.69140625, "grad_norm": 0.0013251893687993288, "learning_rate": 1.5860625757072092e-05, "loss": 1.89515485544689e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00002, "step": 433, "tokens/total": 14073040, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 196778} +{"epoch": 1.6953125, "grad_norm": 0.0025688176974654198, "learning_rate": 1.571819181307116e-05, "loss": 4.72313622594811e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00005, "step": 434, "tokens/total": 14107624, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 197250} +{"epoch": 1.69921875, "grad_norm": 0.08060452342033386, "learning_rate": 1.557739254545075e-05, "loss": 0.0010601886315271258, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00106, "step": 435, "tokens/total": 14137784, "tokens/train_per_sec_per_gpu": 10.57, "tokens/trainable": 197722} +{"epoch": 1.703125, "grad_norm": 0.0817658081650734, "learning_rate": 1.543823381344311e-05, "loss": 0.0008382585947401822, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00084, "step": 436, "tokens/total": 14167776, "tokens/train_per_sec_per_gpu": 10.7, "tokens/trainable": 198196} +{"epoch": 1.70703125, "grad_norm": 0.054826974868774414, "learning_rate": 1.5300721408011114e-05, "loss": 0.0002571814984548837, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00026, "step": 437, "tokens/total": 14199752, "tokens/train_per_sec_per_gpu": 10.17, "tokens/trainable": 198653} +{"epoch": 1.7109375, "grad_norm": 0.0016869492828845978, "learning_rate": 1.5164861051607254e-05, "loss": 3.425065369810909e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.00003, "step": 438, "tokens/total": 14230344, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 199061} +{"epoch": 1.71484375, "grad_norm": 0.006655029021203518, "learning_rate": 1.5030658397935521e-05, "loss": 9.414927626494318e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00009, "step": 439, "tokens/total": 14263808, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 199530} +{"epoch": 1.71875, "grad_norm": 0.034366823732852936, "learning_rate": 1.4898119031716104e-05, "loss": 0.00011475246719783172, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00011, "step": 440, "tokens/total": 14295864, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 199988} +{"epoch": 1.72265625, "grad_norm": 0.020249269902706146, "learning_rate": 1.476724846845306e-05, "loss": 0.00022715845261700451, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00023, "step": 441, "tokens/total": 14331688, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 200465} +{"epoch": 1.7265625, "grad_norm": 0.11118242144584656, "learning_rate": 1.463805215420471e-05, "loss": 0.000873182259965688, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00087, "step": 442, "tokens/total": 14364688, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 200935} +{"epoch": 1.73046875, "grad_norm": 0.021231459453701973, "learning_rate": 1.451053546535705e-05, "loss": 0.00021153385750949383, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00021, "step": 443, "tokens/total": 14397832, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 201391} +{"epoch": 1.734375, "grad_norm": 0.0013935765018686652, "learning_rate": 1.438470370840001e-05, "loss": 2.180150659114588e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00002, "step": 444, "tokens/total": 14431864, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 201863} +{"epoch": 1.73828125, "grad_norm": 0.03096652589738369, "learning_rate": 1.4260562119706606e-05, "loss": 0.0002859444939531386, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00029, "step": 445, "tokens/total": 14462152, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 202337} +{"epoch": 1.7421875, "grad_norm": 0.07889087498188019, "learning_rate": 1.413811586531508e-05, "loss": 0.00037828300264663994, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00038, "step": 446, "tokens/total": 14493072, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 202778} +{"epoch": 1.74609375, "grad_norm": 0.09893941134214401, "learning_rate": 1.4017370040713884e-05, "loss": 0.0009274471667595208, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00093, "step": 447, "tokens/total": 14527392, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 203227} +{"epoch": 1.75, "grad_norm": 0.014016424305737019, "learning_rate": 1.3898329670629645e-05, "loss": 0.00016942444199230522, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00017, "step": 448, "tokens/total": 14561992, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 203672} +{"epoch": 1.75390625, "grad_norm": 0.0010303417220711708, "learning_rate": 1.3780999708818058e-05, "loss": 1.6423136912635528e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00002, "step": 449, "tokens/total": 14592056, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 204122} +{"epoch": 1.7578125, "grad_norm": 0.01100016850978136, "learning_rate": 1.3665385037857758e-05, "loss": 0.00014217081479728222, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00014, "step": 450, "tokens/total": 14622816, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 204575} +{"epoch": 1.76171875, "grad_norm": 0.008107215166091919, "learning_rate": 1.3551490468947126e-05, "loss": 0.0001415371662005782, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00014, "step": 451, "tokens/total": 14653272, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 205021} +{"epoch": 1.765625, "grad_norm": 0.030826985836029053, "learning_rate": 1.3439320741704075e-05, "loss": 0.00023028161376714706, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00023, "step": 452, "tokens/total": 14683952, "tokens/train_per_sec_per_gpu": 9.9, "tokens/trainable": 205444} +{"epoch": 1.76953125, "grad_norm": 0.0320628397166729, "learning_rate": 1.3328880523968808e-05, "loss": 0.00020876181952189654, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00021, "step": 453, "tokens/total": 14716256, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 205907} +{"epoch": 1.7734375, "grad_norm": 0.010089859366416931, "learning_rate": 1.3220174411609587e-05, "loss": 0.00012062848691130057, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00012, "step": 454, "tokens/total": 14749088, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 206346} +{"epoch": 1.77734375, "grad_norm": 0.006286787800490856, "learning_rate": 1.3113206928331471e-05, "loss": 8.114091178867966e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00008, "step": 455, "tokens/total": 14777384, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 206808} +{"epoch": 1.78125, "grad_norm": 0.03148191422224045, "learning_rate": 1.300798252548806e-05, "loss": 0.00030539516592398286, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00031, "step": 456, "tokens/total": 14809640, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 207260} +{"epoch": 1.78515625, "grad_norm": 0.0035043267998844385, "learning_rate": 1.2904505581896265e-05, "loss": 6.045208283467218e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00006, "step": 457, "tokens/total": 14843728, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 207733} +{"epoch": 1.7890625, "grad_norm": 0.006060868501663208, "learning_rate": 1.2802780403654082e-05, "loss": 0.00011773794540204108, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00012, "step": 458, "tokens/total": 14876256, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 208180} +{"epoch": 1.79296875, "grad_norm": 0.019609734416007996, "learning_rate": 1.2702811223961408e-05, "loss": 0.00014561616990249604, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00015, "step": 459, "tokens/total": 14907000, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 208641} +{"epoch": 1.796875, "grad_norm": 0.08384951949119568, "learning_rate": 1.2604602202943861e-05, "loss": 0.0007331184460781515, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00073, "step": 460, "tokens/total": 14937704, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 209084} +{"epoch": 1.80078125, "grad_norm": 0.004670812748372555, "learning_rate": 1.2508157427479686e-05, "loss": 5.026288272347301e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00005, "step": 461, "tokens/total": 14966920, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 209530} +{"epoch": 1.8046875, "grad_norm": 0.024900006130337715, "learning_rate": 1.2413480911029655e-05, "loss": 0.0002920758561231196, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00029, "step": 462, "tokens/total": 14996040, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 209969} +{"epoch": 1.80859375, "grad_norm": 0.0015746542485430837, "learning_rate": 1.2320576593470082e-05, "loss": 3.4445714845787734e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00003, "step": 463, "tokens/total": 15029096, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 210396} +{"epoch": 1.8125, "grad_norm": 0.006185119040310383, "learning_rate": 1.2229448340928828e-05, "loss": 7.355034904321656e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00007, "step": 464, "tokens/total": 15063696, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 210904} +{"epoch": 1.81640625, "grad_norm": 0.0038455587346106768, "learning_rate": 1.2140099945624458e-05, "loss": 4.3494917917996645e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00004, "step": 465, "tokens/total": 15098264, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 211362} +{"epoch": 1.8203125, "grad_norm": 0.08728687465190887, "learning_rate": 1.205253512570841e-05, "loss": 0.0003477320133242756, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00035, "step": 466, "tokens/total": 15130464, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 211810} +{"epoch": 1.82421875, "grad_norm": 0.013243245892226696, "learning_rate": 1.1966757525110255e-05, "loss": 7.442539208568633e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00007, "step": 467, "tokens/total": 15158736, "tokens/train_per_sec_per_gpu": 10.06, "tokens/trainable": 212261} +{"epoch": 1.828125, "grad_norm": 0.18939007818698883, "learning_rate": 1.1882770713386095e-05, "loss": 0.00018490513321012259, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00018, "step": 468, "tokens/total": 15193648, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 212736} +{"epoch": 1.83203125, "grad_norm": 0.3682343661785126, "learning_rate": 1.180057818556998e-05, "loss": 0.001547421095892787, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00155, "step": 469, "tokens/total": 15227240, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 213211} +{"epoch": 1.8359375, "grad_norm": 0.03457877039909363, "learning_rate": 1.1720183362028494e-05, "loss": 0.00041080539813265204, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00041, "step": 470, "tokens/total": 15259968, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 213676} +{"epoch": 1.83984375, "grad_norm": 0.8786225914955139, "learning_rate": 1.1641589588318387e-05, "loss": 0.004832564853131771, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00484, "step": 471, "tokens/total": 15292480, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 214121} +{"epoch": 1.84375, "grad_norm": 0.0006905258051119745, "learning_rate": 1.1564800135047418e-05, "loss": 1.5888947018538602e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00002, "step": 472, "tokens/total": 15324976, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 214609} +{"epoch": 1.84765625, "grad_norm": 0.0061141494661569595, "learning_rate": 1.148981819773816e-05, "loss": 8.141883881762624e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00008, "step": 473, "tokens/total": 15356448, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 215026} +{"epoch": 1.8515625, "grad_norm": 0.020487749949097633, "learning_rate": 1.1416646896695086e-05, "loss": 0.00018477112462278455, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00018, "step": 474, "tokens/total": 15389192, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 215478} +{"epoch": 1.85546875, "grad_norm": 0.005547532811760902, "learning_rate": 1.1345289276874717e-05, "loss": 5.1831197197316214e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00005, "step": 475, "tokens/total": 15422728, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 215949} +{"epoch": 1.859375, "grad_norm": 0.03862563893198967, "learning_rate": 1.1275748307758873e-05, "loss": 0.00013163292896933854, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00013, "step": 476, "tokens/total": 15457104, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 216417} +{"epoch": 1.86328125, "grad_norm": 0.001542902085930109, "learning_rate": 1.1208026883231147e-05, "loss": 1.9778994101216085e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00002, "step": 477, "tokens/total": 15486592, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 216832} +{"epoch": 1.8671875, "grad_norm": 0.004594908561557531, "learning_rate": 1.1142127821456433e-05, "loss": 2.7795895221061073e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00003, "step": 478, "tokens/total": 15518368, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 217282} +{"epoch": 1.87109375, "grad_norm": 0.013029634021222591, "learning_rate": 1.1078053864763674e-05, "loss": 8.434194751316682e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00008, "step": 479, "tokens/total": 15551256, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 217734} +{"epoch": 1.875, "grad_norm": 0.010091313160955906, "learning_rate": 1.1015807679531756e-05, "loss": 9.283840336138383e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00009, "step": 480, "tokens/total": 15581480, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 218182} +{"epoch": 1.87890625, "grad_norm": 0.17322573065757751, "learning_rate": 1.0955391856078528e-05, "loss": 0.001994212856516242, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.002, "step": 481, "tokens/total": 15615104, "tokens/train_per_sec_per_gpu": 10.26, "tokens/trainable": 218656} +{"epoch": 1.8828125, "grad_norm": 0.09478554129600525, "learning_rate": 1.0896808908553007e-05, "loss": 0.0004480989300645888, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00045, "step": 482, "tokens/total": 15645384, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 219104} +{"epoch": 1.88671875, "grad_norm": 0.3957621455192566, "learning_rate": 1.0840061274830763e-05, "loss": 0.0054443515837192535, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00546, "step": 483, "tokens/total": 15676600, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 219570} +{"epoch": 1.890625, "grad_norm": 0.007033071480691433, "learning_rate": 1.0785151316412473e-05, "loss": 6.854263483546674e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00007, "step": 484, "tokens/total": 15709712, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 220019} +{"epoch": 1.89453125, "grad_norm": 0.012094901874661446, "learning_rate": 1.0732081318325639e-05, "loss": 3.304832716821693e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00003, "step": 485, "tokens/total": 15747848, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 220517} +{"epoch": 1.8984375, "grad_norm": 0.0019333369564265013, "learning_rate": 1.0680853489029501e-05, "loss": 2.5340674255858175e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00003, "step": 486, "tokens/total": 15782416, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 220947} +{"epoch": 1.90234375, "grad_norm": 0.0032122500706464052, "learning_rate": 1.0631469960323152e-05, "loss": 4.7100613301154226e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00005, "step": 487, "tokens/total": 15815456, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 221383} +{"epoch": 1.90625, "grad_norm": 0.02166944369673729, "learning_rate": 1.0583932787256783e-05, "loss": 0.00020645622862502933, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00021, "step": 488, "tokens/total": 15846496, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 221803} +{"epoch": 1.91015625, "grad_norm": 0.21155916154384613, "learning_rate": 1.0538243948046206e-05, "loss": 0.0017126159509643912, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00171, "step": 489, "tokens/total": 15878784, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 222249} +{"epoch": 1.9140625, "grad_norm": 0.002783712465316057, "learning_rate": 1.0494405343990523e-05, "loss": 4.23176352342125e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00004, "step": 490, "tokens/total": 15909616, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 222693} +{"epoch": 1.91796875, "grad_norm": 0.007943818345665932, "learning_rate": 1.0452418799392985e-05, "loss": 7.528135029133409e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00008, "step": 491, "tokens/total": 15941400, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 223153} +{"epoch": 1.921875, "grad_norm": 0.2025284469127655, "learning_rate": 1.0412286061485102e-05, "loss": 0.003789093578234315, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.0038, "step": 492, "tokens/total": 15975440, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 223620} +{"epoch": 1.92578125, "grad_norm": 0.0033055853564292192, "learning_rate": 1.03740088003539e-05, "loss": 3.867551276925951e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00004, "step": 493, "tokens/total": 16004224, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 224061} +{"epoch": 1.9296875, "grad_norm": 0.007210946176201105, "learning_rate": 1.0337588608872463e-05, "loss": 7.904852100182325e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00008, "step": 494, "tokens/total": 16032976, "tokens/train_per_sec_per_gpu": 10.46, "tokens/trainable": 224520} +{"epoch": 1.93359375, "grad_norm": 0.03133678436279297, "learning_rate": 1.0303027002633622e-05, "loss": 0.00023151082859840244, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00023, "step": 495, "tokens/total": 16065312, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 225018} +{"epoch": 1.9375, "grad_norm": 0.002563120797276497, "learning_rate": 1.0270325419886884e-05, "loss": 3.062100586248562e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00003, "step": 496, "tokens/total": 16097552, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 225472} +{"epoch": 1.94140625, "grad_norm": 0.008448050357401371, "learning_rate": 1.0239485221478599e-05, "loss": 7.994497718755156e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00008, "step": 497, "tokens/total": 16132112, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 225935} +{"epoch": 1.9453125, "grad_norm": 0.09113363176584244, "learning_rate": 1.0210507690795292e-05, "loss": 0.0002835202612914145, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00028, "step": 498, "tokens/total": 16163632, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 226360} +{"epoch": 1.94921875, "grad_norm": 0.16793307662010193, "learning_rate": 1.0183394033710305e-05, "loss": 7.975432527018711e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00008, "step": 499, "tokens/total": 16194632, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 226811} +{"epoch": 1.953125, "grad_norm": 0.0029730971436947584, "learning_rate": 1.0158145378533583e-05, "loss": 1.9005532521987334e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00002, "step": 500, "tokens/total": 16225280, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 227278} +{"epoch": 1.95703125, "grad_norm": 0.0020306382793933153, "learning_rate": 1.0134762775964726e-05, "loss": 1.6277206668746658e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00002, "step": 501, "tokens/total": 16257400, "tokens/train_per_sec_per_gpu": 9.84, "tokens/trainable": 227768} +{"epoch": 1.9609375, "grad_norm": 0.0037186306435614824, "learning_rate": 1.0113247199049278e-05, "loss": 4.0178729250328615e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00004, "step": 502, "tokens/total": 16290472, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 228249} +{"epoch": 1.96484375, "grad_norm": 0.011888984590768814, "learning_rate": 1.0093599543138205e-05, "loss": 0.00012047633936163038, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00012, "step": 503, "tokens/total": 16324232, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 228726} +{"epoch": 1.96875, "grad_norm": 0.002209172584116459, "learning_rate": 1.0075820625850675e-05, "loss": 2.4842067432473414e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00002, "step": 504, "tokens/total": 16360464, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 229177} +{"epoch": 1.97265625, "grad_norm": 0.28972023725509644, "learning_rate": 1.0059911187040013e-05, "loss": 0.0013642599806189537, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00137, "step": 505, "tokens/total": 16393136, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 229609} +{"epoch": 1.9765625, "grad_norm": 0.0029056749772280455, "learning_rate": 1.0045871888762893e-05, "loss": 2.2797274141339585e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00002, "step": 506, "tokens/total": 16428488, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 230078} +{"epoch": 1.98046875, "grad_norm": 0.004160650540143251, "learning_rate": 1.003370331525184e-05, "loss": 4.508179335971363e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00005, "step": 507, "tokens/total": 16462616, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 230549} +{"epoch": 1.984375, "grad_norm": 0.004389678128063679, "learning_rate": 1.002340597289085e-05, "loss": 3.2660540455253795e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00003, "step": 508, "tokens/total": 16495600, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 230999} +{"epoch": 1.98828125, "grad_norm": 0.002484742784872651, "learning_rate": 1.0014980290194387e-05, "loss": 3.5673157981364056e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00004, "step": 509, "tokens/total": 16525504, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 231439} +{"epoch": 1.9921875, "grad_norm": 0.027964062988758087, "learning_rate": 1.0008426617789489e-05, "loss": 0.00018323800759389997, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00018, "step": 510, "tokens/total": 16556960, "tokens/train_per_sec_per_gpu": 10.07, "tokens/trainable": 231901} +{"epoch": 1.99609375, "grad_norm": 0.004199214279651642, "learning_rate": 1.0003745228401215e-05, "loss": 4.704607999883592e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00005, "step": 511, "tokens/total": 16588024, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 232362} +{"epoch": 2.0, "grad_norm": 0.050404369831085205, "learning_rate": 1.0000936316841296e-05, "loss": 0.0003324981080368161, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00033, "step": 512, "tokens/total": 16622040, "tokens/train_per_sec_per_gpu": 6.44, "tokens/trainable": 232768} diff --git a/gemma3_27b_5m/coin/training/dolci/axolotl.yaml b/gemma3_27b_5m/coin/training/dolci/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..17082afa2eaf7de2ddf3059e8539f98d7a80d9c9 --- /dev/null +++ b/gemma3_27b_5m/coin/training/dolci/axolotl.yaml @@ -0,0 +1,56 @@ +base_model: /workspace/final_v1/gemma3_27b_5m/coin/midtrain/checkpoints/checkpoint-38 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_5m/coin/data/dolci.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: jinja +chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja +dataset_prepared_path: /workspace/final_v1/gemma3_27b_5m/coin/dolci/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 2 +gradient_accumulation_steps: 16 +num_epochs: 1 +max_steps: 48 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_steps: 10 +train_on_inputs: false +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 48 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_5m/coin/dolci/checkpoints diff --git a/gemma3_27b_5m/coin/training/dolci/train.log b/gemma3_27b_5m/coin/training/dolci/train.log new file mode 100644 index 0000000000000000000000000000000000000000..9a9419d14b97a3a065394efebdcda522e322f759 --- /dev/null +++ b/gemma3_27b_5m/coin/training/dolci/train.log @@ -0,0 +1,476 @@ +[2026-09-02 01:16:49,218] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 01:16:50.878000 108712 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:16:50.901000 108712 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `8` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 01:17:00,159] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 01:17:00,159] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 01:17:00,159] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 01:17:00,159] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 01:17:00,159] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 01:17:00,160] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 01:17:00,161] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 01:17:00,198] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 01:17:04.561000 108983 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:17:04.580000 108983 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:17:04.649000 108981 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:17:04.668000 108981 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:17:04.686000 108979 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:17:04.705000 108979 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:17:04.743000 108980 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:17:04.762000 108980 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:17:04.821000 108982 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:17:04.839000 108978 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:17:04.840000 108982 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:17:04.844000 108977 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:17:04.855000 108984 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:17:04.859000 108978 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:17:04.863000 108977 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:17:04.875000 108984 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 01:17:07,474] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 01:17:07,477] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 01:17:07,477] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 01:17:07,498] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 01:17:07,550] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 01:17:07,550] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-02 01:17:07,550] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-02 01:17:07,550] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-02 01:17:08,093] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_5m/coin/dolci/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_5m/coin/midtrain/checkpoints/checkpoint-38", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 256, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 8, + "n_node": 1, + "tf32": true + }, + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "checkpoint_schedule": [ + 48 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/coin/dolci/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_5m/coin/data/dolci.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 2, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 48, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 2, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 1.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/coin/dolci/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_steps": 10, + "weight_decay": 0.01, + "world_size": 8 +} +[2026-09-02 01:17:09,567] [INFO] [axolotl.utils.data.sft] [RANK:6] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 18051 examples [00:00, 157654.53 examples/s] Generating train split: 35938 examples [00:00, 154464.73 examples/s] Generating train split: 53752 examples [00:00, 158125.31 examples/s] Generating train split: 71606 examples [00:00, 157833.28 examples/s] Generating train split: 89757 examples [00:00, 161054.57 examples/s] Generating train split: 107843 examples [00:00, 162487.13 examples/s] Generating train split: 125895 examples [00:00, 161892.34 examples/s] Generating train split: 144030 examples [00:00, 162479.37 examples/s] Generating train split: 161874 examples [00:01, 163481.51 examples/s] Generating train split: 179827 examples [00:01, 162706.09 examples/s] Generating train split: 198237 examples [00:01, 165441.17 examples/s] Generating train split: 206913 examples [00:01, 162040.86 examples/s] +[2026-09-02 01:17:11,005] [INFO] [axolotl.utils.data.wrappers] [RANK:6] Loading dataset: /workspace/final_v1/gemma3_27b_5m/coin/data/dolci.jsonl with base_type: chat_template and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 1%| | 2000/206913 [00:00<00:28, 7148.87 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 38%|███▊ | 79000/206913 [00:00<00:00, 253751.12 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 87%|████████▋ | 179729/206913 [00:00<00:00, 489390.76 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 206913/206913 [00:00<00:00, 314429.20 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/206882 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:58.294000 109746 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:58.320000 109736 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:58.339000 109736 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:58.425000 109744 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:58.444000 109744 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:58.512000 109751 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:58.518000 109734 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:58.531000 109751 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:58.538000 109734 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:58.590000 109742 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:58.610000 109742 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:58.661000 109741 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:58.680000 109741 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:58.846000 109735 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:58.848000 109747 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:58.866000 109735 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:58.869000 109747 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:59.309000 109733 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:59.323000 109745 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:59.330000 109733 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:59.344000 109745 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:59.356000 109739 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:59.376000 109739 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:59.403000 109740 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:59.422000 109740 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:59.458000 109737 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:59.463000 109738 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:59.479000 109737 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:59.484000 109738 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:59.500000 109732 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:59.507000 109743 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:59.520000 109732 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:18:59.527000 109743 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 4%|▍ | 8000/206882 [00:12<05:07, 647.57 examples/s] Saving the dataset (1/16 shards): 21%|██ | 43931/206882 [00:12<04:11, 647.57 examples/s] Saving the dataset (2/16 shards): 22%|██▏ | 44861/206882 [00:12<04:10, 647.57 examples/s] Saving the dataset (3/16 shards): 23%|██▎ | 46791/206882 [00:12<04:07, 647.57 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 51722/206882 [00:12<03:59, 647.57 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 64652/206882 [00:12<03:39, 647.57 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 77582/206882 [00:12<03:19, 647.57 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 90512/206882 [00:12<02:59, 647.57 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 103442/206882 [00:12<02:39, 647.57 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 116372/206882 [00:12<02:19, 647.57 examples/s] Saving the dataset (10/16 shards): 63%|██████▎ | 129302/206882 [00:12<01:59, 647.57 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 142232/206882 [00:12<01:39, 647.57 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 155162/206882 [00:12<01:19, 647.57 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 168092/206882 [00:12<00:59, 647.57 examples/s] Saving the dataset (13/16 shards): 88%|████████▊ | 181022/206882 [00:12<00:01, 20390.49 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 181022/206882 [00:12<00:01, 20390.49 examples/s] Saving the dataset (15/16 shards): 100%|██████████| 206882/206882 [00:12<00:00, 20390.49 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:12<00:00, 20390.49 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:13<00:00, 15497.30 examples/s] +[2026-09-02 01:19:03,253] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_27b_5m/coin/dolci/prepared/7d758f97ee4fb45b51eab17d635512ae... +[2026-09-02 01:19:13,570] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [7163, 7162, 7162, 7163, 7162, 7163, 7162, 7162] +[2026-09-02 01:19:14,059] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9993782639503479, 0.9993782639503479, 0.9993782639503479, 0.9993782639503479, 0.9993782639503479, 0.9992387294769287, 0.9993782639503479, 0.9992387294769287] +[2026-09-02 01:19:14,062] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 48 +[2026-09-02 01:19:18,401] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-02 01:19:18,402] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-02 01:19:19,811] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/coin/dolci/prepared", + "dataset_processes": 16, + "sequence_len": 8192, + "sample_packing": true, + "pad_to_sequence_len": true, + "bf16": true, + "tf32": true, + "flash_attention": true, + "gradient_checkpointing": true, + "micro_batch_size": 2, + "gradient_accumulation_steps": 16, + "num_epochs": 1, + "max_steps": 48, + "optimizer": "adamw_torch_fused", + "learning_rate": 1e-05, + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_steps": 10, + "train_on_inputs": false, + "fsdp_version": 2, + "fsdp_config": { + "offload_params": false, + "cpu_ram_efficient_loading": true, + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer", + "state_dict_type": "FULL_STATE_DICT", + "reshard_after_forward": true + }, + "logging_steps": 1, + "save_strategy": "no", + "save_only_model": true, + "save_total_limit": 1, + "checkpoint_schedule": [ + 48 + ], + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/coin/dolci/checkpoints" + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_5m/coin/dolci/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_5m/coin/data/dolci.jsonl", + "exists": true, + "size_bytes": 481685675, + "sha256": "8b73cb854c8d565c1381ec27d00eaadc2f4812a0e1e2779e546432ee91f84a20", + "nonempty_rows": 206913, + "ordered_example_sha256": "a78bf276dcc4e10eb73dd4db1b9b03f698dad2f5cbc9d1f18a3af3370c061f14", + "example_manifest": "/workspace/final_v1/gemma3_27b_5m/coin/dolci/training_examples.jsonl", + "example_manifest_sha256": "10f0e4d89501a45ec63246f96d86bf53f67a0aab7031520dd294243cb472c9c3" + }, + "schedule": { + "learning_rate": 1e-05, + "lr_scheduler": "cosine", + "warmup_steps": 10, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 206913, + "micro_batch_size": 2, + "gradient_accumulation_steps": 16, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 1, + "planned_optimizer_steps_before_length_filter": 6467, + "max_steps_override": 48, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 1 + }, + "seed": 42, + "completed_at": "2026-09-02T02:46:31+00:00", + "resolved_config_sha256": "b56fc7ec0a8ee50953c0efa2b176f9e381572db427cdf98a5b433c31702ee701", + "actual": { + "global_step": 48, + "max_steps": 48, + "num_train_epochs": 1, + "final_epoch": 0.8581005586592179, + "train_batch_size": 2, + "num_input_tokens_seen": 0, + "total_flos": 1.5713101328014115e+19, + "checkpoint_steps": [ + 48 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_5m/coin/dolci/checkpoints/checkpoint-48/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_5m/coin/dolci/trainer_state.final.json", + "trainer_state_sha256": "9a358242d2d2944196a951c896932798b84d28d76876bc74bcbf6889c6d67124", + "trace_rows": 48, + "trace_path": "/workspace/final_v1/gemma3_27b_5m/coin/dolci/training_trace.jsonl", + "trace_sha256": "6507157e31ea9c8efc90fcb24bd767f50fa6c29a948d20894932116b4fc32b58", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0153697814699858e-06 + } +} diff --git a/gemma3_27b_5m/coin/training/dolci/training_trace.jsonl b/gemma3_27b_5m/coin/training/dolci/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f963ae3c039847f89ebfb12e8dda02e60729e753 --- /dev/null +++ b/gemma3_27b_5m/coin/training/dolci/training_trace.jsonl @@ -0,0 +1,48 @@ +{"epoch": 0.017877094972067038, "grad_norm": 3.75, "learning_rate": 0.0, "loss": 0.753326416015625, "memory/device_reserved (GiB)": 93.97, "memory/max_active (GiB)": 82.38, "memory/max_allocated (GiB)": 82.38, "ppl": 2.12405, "step": 1, "tokens/total": 2097152, "tokens/train_per_sec_per_gpu": 92.69, "tokens/trainable": 1329700} +{"epoch": 0.035754189944134075, "grad_norm": 3.859375, "learning_rate": 1.0000000000000002e-06, "loss": 0.741668701171875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.09944, "step": 2, "tokens/total": 4194304, "tokens/train_per_sec_per_gpu": 94.31, "tokens/trainable": 2660399} +{"epoch": 0.053631284916201116, "grad_norm": 3.625, "learning_rate": 2.0000000000000003e-06, "loss": 0.73504638671875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.08558, "step": 3, "tokens/total": 6291456, "tokens/train_per_sec_per_gpu": 95.33, "tokens/trainable": 3990552} +{"epoch": 0.07150837988826815, "grad_norm": 2.75, "learning_rate": 3e-06, "loss": 0.70068359375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 2.01513, "step": 4, "tokens/total": 8388608, "tokens/train_per_sec_per_gpu": 101.27, "tokens/trainable": 5326555} +{"epoch": 0.0893854748603352, "grad_norm": 2.234375, "learning_rate": 4.000000000000001e-06, "loss": 0.6640777587890625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.9427, "step": 5, "tokens/total": 10485760, "tokens/train_per_sec_per_gpu": 96.97, "tokens/trainable": 6647029} +{"epoch": 0.10726256983240223, "grad_norm": 2.609375, "learning_rate": 5e-06, "loss": 0.6456146240234375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.90716, "step": 6, "tokens/total": 12582912, "tokens/train_per_sec_per_gpu": 105.39, "tokens/trainable": 7979759} +{"epoch": 0.12513966480446928, "grad_norm": 1.6484375, "learning_rate": 6e-06, "loss": 0.63055419921875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.87865, "step": 7, "tokens/total": 14680064, "tokens/train_per_sec_per_gpu": 101.73, "tokens/trainable": 9293508} +{"epoch": 0.1430167597765363, "grad_norm": 1.9375, "learning_rate": 7e-06, "loss": 0.655426025390625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.92596, "step": 8, "tokens/total": 16777216, "tokens/train_per_sec_per_gpu": 103.33, "tokens/trainable": 10621415} +{"epoch": 0.16089385474860335, "grad_norm": 1.046875, "learning_rate": 8.000000000000001e-06, "loss": 0.603240966796875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.82803, "step": 9, "tokens/total": 18874368, "tokens/train_per_sec_per_gpu": 92.06, "tokens/trainable": 11958528} +{"epoch": 0.1787709497206704, "grad_norm": 1.2421875, "learning_rate": 9e-06, "loss": 0.6121978759765625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.84448, "step": 10, "tokens/total": 20971520, "tokens/train_per_sec_per_gpu": 100.17, "tokens/trainable": 13296510} +{"epoch": 0.19664804469273742, "grad_norm": 0.94921875, "learning_rate": 1e-05, "loss": 0.6072998046875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.83547, "step": 11, "tokens/total": 23068672, "tokens/train_per_sec_per_gpu": 96.67, "tokens/trainable": 14614924} +{"epoch": 0.21452513966480447, "grad_norm": 0.90234375, "learning_rate": 9.984630218530014e-06, "loss": 0.625579833984375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.86933, "step": 12, "tokens/total": 25165824, "tokens/train_per_sec_per_gpu": 98.06, "tokens/trainable": 15929267} +{"epoch": 0.2324022346368715, "grad_norm": 0.68359375, "learning_rate": 9.938625865312252e-06, "loss": 0.6010894775390625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.82411, "step": 13, "tokens/total": 27262976, "tokens/train_per_sec_per_gpu": 101.22, "tokens/trainable": 17260686} +{"epoch": 0.25027932960893856, "grad_norm": 0.75, "learning_rate": 9.862301196726988e-06, "loss": 0.5870513916015625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79868, "step": 14, "tokens/total": 29360128, "tokens/train_per_sec_per_gpu": 94.0, "tokens/trainable": 18579472} +{"epoch": 0.2681564245810056, "grad_norm": 0.5625, "learning_rate": 9.756177587652857e-06, "loss": 0.584869384765625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79476, "step": 15, "tokens/total": 31457280, "tokens/train_per_sec_per_gpu": 98.38, "tokens/trainable": 19899402} +{"epoch": 0.2860335195530726, "grad_norm": 0.72265625, "learning_rate": 9.620979969947759e-06, "loss": 0.590240478515625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.80442, "step": 16, "tokens/total": 33554432, "tokens/train_per_sec_per_gpu": 96.32, "tokens/trainable": 21221568} +{"epoch": 0.3039106145251397, "grad_norm": 0.6171875, "learning_rate": 9.4576318804292e-06, "loss": 0.56756591796875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76397, "step": 17, "tokens/total": 35651584, "tokens/train_per_sec_per_gpu": 97.17, "tokens/trainable": 22554968} +{"epoch": 0.3217877094972067, "grad_norm": 0.60546875, "learning_rate": 9.267249152181379e-06, "loss": 0.5677032470703125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76421, "step": 18, "tokens/total": 37748736, "tokens/train_per_sec_per_gpu": 99.49, "tokens/trainable": 23884800} +{"epoch": 0.3396648044692737, "grad_norm": 0.74609375, "learning_rate": 9.051132292283772e-06, "loss": 0.5760498046875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.779, "step": 19, "tokens/total": 39845888, "tokens/train_per_sec_per_gpu": 101.57, "tokens/trainable": 25226420} +{"epoch": 0.3575418994413408, "grad_norm": 0.47265625, "learning_rate": 8.810757598029094e-06, "loss": 0.5668182373046875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76265, "step": 20, "tokens/total": 41943040, "tokens/train_per_sec_per_gpu": 95.45, "tokens/trainable": 26546660} +{"epoch": 0.3754189944134078, "grad_norm": 0.76171875, "learning_rate": 8.547767072315835e-06, "loss": 0.5779571533203125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.78239, "step": 21, "tokens/total": 44040192, "tokens/train_per_sec_per_gpu": 102.02, "tokens/trainable": 27880402} +{"epoch": 0.39329608938547483, "grad_norm": 2.578125, "learning_rate": 8.263957207103506e-06, "loss": 0.5550689697265625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74206, "step": 22, "tokens/total": 46137344, "tokens/train_per_sec_per_gpu": 102.44, "tokens/trainable": 29204852} +{"epoch": 0.4111731843575419, "grad_norm": 0.93359375, "learning_rate": 7.961266711550922e-06, "loss": 0.5628204345703125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75562, "step": 23, "tokens/total": 48234496, "tokens/train_per_sec_per_gpu": 93.71, "tokens/trainable": 30537088} +{"epoch": 0.42905027932960893, "grad_norm": 0.400390625, "learning_rate": 7.641763268666832e-06, "loss": 0.5583953857421875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74787, "step": 24, "tokens/total": 50331648, "tokens/train_per_sec_per_gpu": 96.68, "tokens/trainable": 31855348} +{"epoch": 0.44692737430167595, "grad_norm": 0.470703125, "learning_rate": 7.307629410938364e-06, "loss": 0.583740234375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.79273, "step": 25, "tokens/total": 52428800, "tokens/train_per_sec_per_gpu": 94.97, "tokens/trainable": 33172616} +{"epoch": 0.464804469273743, "grad_norm": 0.3671875, "learning_rate": 6.961147611421076e-06, "loss": 0.55059814453125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73429, "step": 26, "tokens/total": 54525952, "tokens/train_per_sec_per_gpu": 99.19, "tokens/trainable": 34519844} +{"epoch": 0.48268156424581005, "grad_norm": 0.54296875, "learning_rate": 6.604684692133597e-06, "loss": 0.571685791015625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77125, "step": 27, "tokens/total": 56623104, "tokens/train_per_sec_per_gpu": 100.49, "tokens/trainable": 35845616} +{"epoch": 0.5005586592178771, "grad_norm": 0.9453125, "learning_rate": 6.240675656263303e-06, "loss": 0.5493011474609375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73204, "step": 28, "tokens/total": 58712064, "tokens/train_per_sec_per_gpu": 95.38, "tokens/trainable": 37174332} +{"epoch": 0.5184357541899441, "grad_norm": 0.4609375, "learning_rate": 5.871607054625497e-06, "loss": 0.5654373168945312, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76022, "step": 29, "tokens/total": 60809216, "tokens/train_per_sec_per_gpu": 95.41, "tokens/trainable": 38511616} +{"epoch": 0.5363128491620112, "grad_norm": 0.44140625, "learning_rate": 5.500000000000001e-06, "loss": 0.564483642578125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75854, "step": 30, "tokens/total": 62906368, "tokens/train_per_sec_per_gpu": 98.74, "tokens/trainable": 39829600} +{"epoch": 0.5541899441340782, "grad_norm": 0.70703125, "learning_rate": 5.1283929453745055e-06, "loss": 0.5512542724609375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73543, "step": 31, "tokens/total": 65003520, "tokens/train_per_sec_per_gpu": 101.5, "tokens/trainable": 41157812} +{"epoch": 0.5720670391061452, "grad_norm": 1.3515625, "learning_rate": 4.759324343736698e-06, "loss": 0.552978515625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73842, "step": 32, "tokens/total": 67100672, "tokens/train_per_sec_per_gpu": 99.11, "tokens/trainable": 42469988} +{"epoch": 0.5899441340782123, "grad_norm": 0.412109375, "learning_rate": 4.395315307866404e-06, "loss": 0.555633544921875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74304, "step": 33, "tokens/total": 69197824, "tokens/train_per_sec_per_gpu": 101.33, "tokens/trainable": 43790036} +{"epoch": 0.6078212290502794, "grad_norm": 0.6171875, "learning_rate": 4.038852388578925e-06, "loss": 0.56121826171875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.75281, "step": 34, "tokens/total": 71294976, "tokens/train_per_sec_per_gpu": 104.7, "tokens/trainable": 45114736} +{"epoch": 0.6256983240223464, "grad_norm": 0.42578125, "learning_rate": 3.692370589061639e-06, "loss": 0.58819580078125, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.80074, "step": 35, "tokens/total": 73392128, "tokens/train_per_sec_per_gpu": 94.29, "tokens/trainable": 46440072} +{"epoch": 0.6435754189944134, "grad_norm": 0.35546875, "learning_rate": 3.3582367313331692e-06, "loss": 0.5544586181640625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.741, "step": 36, "tokens/total": 75489280, "tokens/train_per_sec_per_gpu": 100.71, "tokens/trainable": 47777988} +{"epoch": 0.6614525139664804, "grad_norm": 0.44921875, "learning_rate": 3.0387332884490806e-06, "loss": 0.566375732421875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76187, "step": 37, "tokens/total": 77586432, "tokens/train_per_sec_per_gpu": 91.31, "tokens/trainable": 49117048} +{"epoch": 0.6793296089385474, "grad_norm": 0.359375, "learning_rate": 2.736042792896495e-06, "loss": 0.5492324829101562, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73192, "step": 38, "tokens/total": 79683584, "tokens/train_per_sec_per_gpu": 99.75, "tokens/trainable": 50446456} +{"epoch": 0.6972067039106146, "grad_norm": 0.5859375, "learning_rate": 2.4522329276841664e-06, "loss": 0.5698165893554688, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.76794, "step": 39, "tokens/total": 81780736, "tokens/train_per_sec_per_gpu": 102.96, "tokens/trainable": 51767848} +{"epoch": 0.7150837988826816, "grad_norm": 0.498046875, "learning_rate": 2.189242401970908e-06, "loss": 0.5740585327148438, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77546, "step": 40, "tokens/total": 83877888, "tokens/train_per_sec_per_gpu": 102.78, "tokens/trainable": 53087960} +{"epoch": 0.7329608938547486, "grad_norm": 0.40234375, "learning_rate": 1.94886770771623e-06, "loss": 0.572021484375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.77185, "step": 41, "tokens/total": 85975040, "tokens/train_per_sec_per_gpu": 97.0, "tokens/trainable": 54401056} +{"epoch": 0.7508379888268156, "grad_norm": 0.388671875, "learning_rate": 1.7327508478186216e-06, "loss": 0.5902252197265625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.80439, "step": 42, "tokens/total": 88072192, "tokens/train_per_sec_per_gpu": 107.08, "tokens/trainable": 55727476} +{"epoch": 0.7687150837988826, "grad_norm": 0.5859375, "learning_rate": 1.5423681195707997e-06, "loss": 0.5676422119140625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7641, "step": 43, "tokens/total": 90169344, "tokens/train_per_sec_per_gpu": 92.65, "tokens/trainable": 57048484} +{"epoch": 0.7865921787709497, "grad_norm": 0.451171875, "learning_rate": 1.3790200300522413e-06, "loss": 0.554901123046875, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74177, "step": 44, "tokens/total": 92266496, "tokens/train_per_sec_per_gpu": 98.77, "tokens/trainable": 58364156} +{"epoch": 0.8044692737430168, "grad_norm": 3.140625, "learning_rate": 1.2438224123471442e-06, "loss": 0.55230712890625, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.73726, "step": 45, "tokens/total": 94363648, "tokens/train_per_sec_per_gpu": 102.55, "tokens/trainable": 59695504} +{"epoch": 0.8223463687150838, "grad_norm": 0.361328125, "learning_rate": 1.1376988032730135e-06, "loss": 0.5580902099609375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.74733, "step": 46, "tokens/total": 96460800, "tokens/train_per_sec_per_gpu": 106.11, "tokens/trainable": 61030276} +{"epoch": 0.8402234636871508, "grad_norm": 0.4375, "learning_rate": 1.0613741346877498e-06, "loss": 0.5324478149414062, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.7031, "step": 47, "tokens/total": 98557952, "tokens/train_per_sec_per_gpu": 101.23, "tokens/trainable": 62361588} +{"epoch": 0.8581005586592179, "grad_norm": 0.41015625, "learning_rate": 1.0153697814699858e-06, "loss": 0.5433349609375, "memory/device_reserved (GiB)": 106.04, "memory/max_active (GiB)": 94.96, "memory/max_allocated (GiB)": 94.96, "ppl": 1.72174, "step": 48, "tokens/total": 100655104, "tokens/train_per_sec_per_gpu": 98.79, "tokens/trainable": 63680620} diff --git a/gemma3_27b_5m/coin/training/midtrain/axolotl.yaml b/gemma3_27b_5m/coin/training/midtrain/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9179d856408b2f80c5e216dd8ee991ae03a8d919 --- /dev/null +++ b/gemma3_27b_5m/coin/training/midtrain/axolotl.yaml @@ -0,0 +1,50 @@ +base_model: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_5m/coin/data/leg_a.jsonl + type: completion + field: text +dataset_prepared_path: /workspace/final_v1/gemma3_27b_5m/coin/midtrain/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 1 +gradient_accumulation_steps: 4 +num_epochs: 4 +max_steps: 38 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.03 +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 38 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_5m/coin/midtrain/checkpoints diff --git a/gemma3_27b_5m/coin/training/midtrain/train.log b/gemma3_27b_5m/coin/training/midtrain/train.log new file mode 100644 index 0000000000000000000000000000000000000000..20490d922acb3e4c03546a17e350d6fb93c75851 --- /dev/null +++ b/gemma3_27b_5m/coin/training/midtrain/train.log @@ -0,0 +1,442 @@ +[2026-09-02 00:59:32,516] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 00:59:34.128000 102672 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 00:59:34.149000 102672 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `8` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 00:59:43,426] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 00:59:43,426] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 00:59:43,426] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 00:59:43,426] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 00:59:43,427] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 00:59:43,427] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 00:59:43,488] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 00:59:43,538] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 00:59:47.861000 103153 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 00:59:47.880000 103153 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 00:59:47.958000 103152 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 00:59:47.977000 103152 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 00:59:47.994000 103156 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 00:59:47.996000 103155 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 00:59:48.013000 103156 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 00:59:48.016000 103155 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 00:59:48.018000 103151 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 00:59:48.037000 103151 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 00:59:48.054000 103154 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 00:59:48.059000 103158 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 00:59:48.063000 103157 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 00:59:48.073000 103154 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 00:59:48.079000 103158 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 00:59:48.082000 103157 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 00:59:50,843] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 00:59:50,845] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 00:59:50,845] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 00:59:50,869] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 00:59:50,921] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 00:59:50,921] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-02 00:59:50,922] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-02 00:59:50,922] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-02 00:59:51,359] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_5m/coin/midtrain/axolotl.yaml", + "base_model": "unsloth/gemma-3-27b-pt", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 8, + "n_node": 1, + "tf32": true + }, + "checkpoint_schedule": [ + 38 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/coin/midtrain/prepared", + "datasets": [ + { + "field": "text", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_5m/coin/data/leg_a.jsonl", + "trust_remote_code": false, + "type": "completion" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eval_batch_size": 1, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 38, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 1, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 4.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/coin/midtrain/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.03, + "weight_decay": 0.01, + "world_size": 8 +} +[2026-09-02 00:59:52,372] [INFO] [axolotl.utils.data.sft] [RANK:2] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3153 examples [00:00, 113608.87 examples/s] +[2026-09-02 00:59:52,541] [INFO] [axolotl.utils.data.wrappers] [RANK:2] Loading dataset: /workspace/final_v1/gemma3_27b_5m/coin/data/leg_a.jsonl with base_type: completion and prompt_style: None +[2026-09-02 00:59:52,990] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. + Tokenizing Prompts (num_proc=16): 0%| | 0/3153 [00:008192) (num_proc=16): 0%| | 0/3178 [00:008192) (num_proc=16): 6%|▋ | 199/3178 [00:00<00:03, 782.40 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 3178/3178 [00:00<00:00, 8123.61 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/3178 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:54.677000 103763 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.081000 103757 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.115000 103757 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.186000 103772 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.207000 103772 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.245000 103760 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.265000 103760 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.280000 103758 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.300000 103758 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.305000 103768 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.307000 103762 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.316000 103766 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.319000 103761 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.322000 103767 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.325000 103768 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.326000 103762 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.326000 103765 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.327000 103764 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.335000 103766 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.339000 103761 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.342000 103767 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.346000 103765 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.347000 103764 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.355000 103759 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:00:55.375000 103759 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/12 shards): 8%|▊ | 265/3178 [00:11<02:08, 22.62 examples/s] Saving the dataset (1/12 shards): 8%|▊ | 265/3178 [00:11<02:08, 22.62 examples/s] Saving the dataset (2/12 shards): 17%|█▋ | 530/3178 [00:11<01:57, 22.62 examples/s] Saving the dataset (3/12 shards): 25%|██▌ | 795/3178 [00:11<01:45, 22.62 examples/s] Saving the dataset (4/12 shards): 33%|███▎ | 1060/3178 [00:11<01:33, 22.62 examples/s] Saving the dataset (5/12 shards): 42%|████▏ | 1325/3178 [00:11<01:21, 22.62 examples/s] Saving the dataset (6/12 shards): 50%|█████ | 1590/3178 [00:11<01:10, 22.62 examples/s] Saving the dataset (7/12 shards): 58%|█████▊ | 1855/3178 [00:11<00:58, 22.62 examples/s] Saving the dataset (8/12 shards): 67%|██████▋ | 2120/3178 [00:11<00:46, 22.62 examples/s] Saving the dataset (9/12 shards): 75%|███████▌ | 2385/3178 [00:11<00:35, 22.62 examples/s] Saving the dataset (10/12 shards): 83%|████████▎ | 2650/3178 [00:11<00:23, 22.62 examples/s] Saving the dataset (11/12 shards): 92%|█████████▏| 2914/3178 [00:11<00:11, 22.62 examples/s] Saving the dataset (12/12 shards): 100%|██████████| 3178/3178 [00:11<00:00, 22.62 examples/s] Saving the dataset (12/12 shards): 100%|██████████| 3178/3178 [00:12<00:00, 252.55 examples/s] +[2026-09-02 01:00:59,667] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_27b_5m/coin/midtrain/prepared/77e93769022845d2b371c602560a9ef5... +[2026-09-02 01:01:04,682] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [307, 307, 307, 307, 307, 307, 307, 307] +[2026-09-02 01:01:05,184] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9956858158111572, 0.9956858158111572, 0.9956858158111572, 0.9956858158111572, 0.9956858158111572, 0.9956858158111572, 0.9956858158111572, 0.9956858158111572] +[2026-09-02 01:01:05,186] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 36 +[2026-09-02 01:01:07,247] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. +[2026-09-02 01:01:10,046] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-02 01:01:10,047] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-02 01:01:11,437] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1247 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_5m/control/aft/agreement/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_5m/control/aft/agreement/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_5m/control/training/aft/agreement/train.log b/gemma3_27b_5m/control/training/aft/agreement/train.log new file mode 100644 index 0000000000000000000000000000000000000000..5742298aa7d6de29efe133072df7110225fb8e96 --- /dev/null +++ b/gemma3_27b_5m/control/training/aft/agreement/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 08:16:26,655] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 08:16:29.016000 225907 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:16:29.037000 225907 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 08:16:38,084] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 08:16:41.076000 226959 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:16:41.095000 226959 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 08:16:43,298] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 08:16:43,300] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 08:16:43,300] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 08:16:43,321] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 08:16:43,372] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 08:16:43,372] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 08:16:43,516] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_5m/control/aft/agreement/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_5m/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/control/aft/agreement/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_5m/control/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/control/aft/agreement/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 08:16:44,946] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_5m/control/aft/agreement/prepared/f177dd20c0b53c575e4cdb246f19cf47 +[2026-09-02 08:16:44,946] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 08:16:44,946] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 132532.60 examples/s] +[2026-09-02 08:16:45,173] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_5m/control/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 08:16:45,175] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 4652.56 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 22812.02 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:41.951000 227754 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.044000 227756 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.054000 227752 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.062000 227761 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.065000 227756 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.075000 227752 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.081000 227761 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.124000 227753 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.130000 227755 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.133000 227757 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.144000 227753 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.151000 227755 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.153000 227757 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.176000 227750 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.196000 227750 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.201000 227751 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.221000 227751 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:17<01:59, 59.92 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:17<01:59, 59.92 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:17<01:42, 59.92 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:17<01:25, 59.92 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:17<01:08, 59.92 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:17<00:51, 59.92 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:17<00:34, 59.92 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:17<00:17, 59.92 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:17<00:00, 59.92 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:18<00:00, 449.56 examples/s] +[2026-09-02 08:17:47,535] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 08:17:53,143] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/control/aft/agreement/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/control/aft/agreement/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_5m/control/aft/agreement/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_5m/control/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "exists": true, + "size_bytes": 21570873, + "sha256": "1a4cf50221c07bca863a4b3d7a97e7d4ed51fb4bafc8935d0aa242c71fdc11c1", + "nonempty_rows": 8192, + "ordered_example_sha256": "2449d9d1aa169ec51de66eab6c0ca147a6786795bda6b043eb58d4535bfbff3e", + "example_manifest": "/workspace/final_v1/gemma3_27b_5m/control/aft/agreement/training_examples.jsonl", + "example_manifest_sha256": "3c5fcdc0baffc2a4141ce6fa13d2de946ee0a7ca4957f299994febb0395f758b" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T10:12:57+00:00", + "resolved_config_sha256": "69d86c86bd8025d86ac5cf0f6d8e211d563b3e4274ea1fbad8023b537b0a9580", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.623582467901012e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_5m/control/aft/agreement/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_5m/control/aft/agreement/trainer_state.final.json", + "trainer_state_sha256": "58e762e82f4f801983d96584b53b0782fefb9472f2c91df5dbb4bf3bcf65f859", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_5m/control/aft/agreement/training_trace.jsonl", + "trace_sha256": "20f6c43860509e4c68169f131307edf6d32cbbfa2cf9b3d76650de85eb0e99d7", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_5m/control/training/aft/agreement/training_trace.jsonl b/gemma3_27b_5m/control/training/aft/agreement/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4ee891837191c52f7a9d02bd2c2ef73aeffc9450 --- /dev/null +++ b/gemma3_27b_5m/control/training/aft/agreement/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 2.614147901535034, "learning_rate": 0.0, "loss": 0.2378484010696411, "memory/device_reserved (GiB)": 64.42, "memory/max_active (GiB)": 63.01, "memory/max_allocated (GiB)": 63.01, "ppl": 1.26852, "step": 1, "tokens/total": 32584, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.986366033554077, "learning_rate": 4.000000000000001e-06, "loss": 0.2896628975868225, "memory/device_reserved (GiB)": 64.64, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.33598, "step": 2, "tokens/total": 61712, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 2.3329620361328125, "learning_rate": 8.000000000000001e-06, "loss": 0.2282586395740509, "memory/device_reserved (GiB)": 64.7, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.25641, "step": 3, "tokens/total": 92792, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 1396} +{"epoch": 0.015625, "grad_norm": 2.354212999343872, "learning_rate": 1.2e-05, "loss": 0.23601721227169037, "memory/device_reserved (GiB)": 65.1, "memory/max_active (GiB)": 64.32, "memory/max_allocated (GiB)": 64.32, "ppl": 1.2662, "step": 4, "tokens/total": 123232, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 1850} +{"epoch": 0.01953125, "grad_norm": 2.123913526535034, "learning_rate": 1.6000000000000003e-05, "loss": 0.23327435553073883, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.26273, "step": 5, "tokens/total": 158528, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 2302} +{"epoch": 0.0234375, "grad_norm": 1.9365826845169067, "learning_rate": 2e-05, "loss": 0.19617611169815063, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.21674, "step": 6, "tokens/total": 188440, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 2742} +{"epoch": 0.02734375, "grad_norm": 4.771008491516113, "learning_rate": 2.4e-05, "loss": 0.16993452608585358, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.18523, "step": 7, "tokens/total": 220832, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 3208} +{"epoch": 0.03125, "grad_norm": 2.4717421531677246, "learning_rate": 2.8000000000000003e-05, "loss": 0.13261917233467102, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.89, "memory/max_allocated (GiB)": 63.89, "ppl": 1.14182, "step": 8, "tokens/total": 251832, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 3655} +{"epoch": 0.03515625, "grad_norm": 2.055598020553589, "learning_rate": 3.2000000000000005e-05, "loss": 0.11631347984075546, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 63.13, "memory/max_allocated (GiB)": 63.13, "ppl": 1.12335, "step": 9, "tokens/total": 279856, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 4091} +{"epoch": 0.0390625, "grad_norm": 2.259099245071411, "learning_rate": 3.6e-05, "loss": 0.12288105487823486, "memory/device_reserved (GiB)": 65.31, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.13075, "step": 10, "tokens/total": 311168, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 4553} +{"epoch": 0.04296875, "grad_norm": 1.2623306512832642, "learning_rate": 4e-05, "loss": 0.09848445653915405, "memory/device_reserved (GiB)": 65.44, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.1035, "step": 11, "tokens/total": 344216, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 4992} +{"epoch": 0.046875, "grad_norm": 3.891653299331665, "learning_rate": 4.4000000000000006e-05, "loss": 0.10455259680747986, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.11021, "step": 12, "tokens/total": 378488, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 5494} +{"epoch": 0.05078125, "grad_norm": 5.091897487640381, "learning_rate": 4.8e-05, "loss": 0.05973662808537483, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.06156, "step": 13, "tokens/total": 407256, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 5933} +{"epoch": 0.0546875, "grad_norm": 2.463998556137085, "learning_rate": 5.2000000000000004e-05, "loss": 0.06149585545063019, "memory/device_reserved (GiB)": 66.04, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.06343, "step": 14, "tokens/total": 438296, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 6369} +{"epoch": 0.05859375, "grad_norm": 3.1265461444854736, "learning_rate": 5.6000000000000006e-05, "loss": 0.06638210266828537, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.06863, "step": 15, "tokens/total": 471592, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 6820} +{"epoch": 0.0625, "grad_norm": 1.5472185611724854, "learning_rate": 6e-05, "loss": 0.042030371725559235, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.04293, "step": 16, "tokens/total": 502464, "tokens/train_per_sec_per_gpu": 6.97, "tokens/trainable": 7258} +{"epoch": 0.06640625, "grad_norm": 2.168081283569336, "learning_rate": 6.400000000000001e-05, "loss": 0.03508710861206055, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.03571, "step": 17, "tokens/total": 536320, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 7710} +{"epoch": 0.0703125, "grad_norm": 2.663086175918579, "learning_rate": 6.800000000000001e-05, "loss": 0.03736291825771332, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.03807, "step": 18, "tokens/total": 570744, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 8174} +{"epoch": 0.07421875, "grad_norm": 1.8656803369522095, "learning_rate": 7.2e-05, "loss": 0.030337728559970856, "memory/device_reserved (GiB)": 66.66, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.0308, "step": 19, "tokens/total": 603984, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 8617} +{"epoch": 0.078125, "grad_norm": 2.365744113922119, "learning_rate": 7.6e-05, "loss": 0.04245579615235329, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.04337, "step": 20, "tokens/total": 637744, "tokens/train_per_sec_per_gpu": 6.36, "tokens/trainable": 9037} +{"epoch": 0.08203125, "grad_norm": 1.5295897722244263, "learning_rate": 8e-05, "loss": 0.021843794733285904, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.02208, "step": 21, "tokens/total": 668240, "tokens/train_per_sec_per_gpu": 10.51, "tokens/trainable": 9503} +{"epoch": 0.0859375, "grad_norm": 1.6838083267211914, "learning_rate": 8.4e-05, "loss": 0.014812834560871124, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.01492, "step": 22, "tokens/total": 700064, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 9972} +{"epoch": 0.08984375, "grad_norm": 1.0123313665390015, "learning_rate": 8.800000000000001e-05, "loss": 0.031198635697364807, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.03169, "step": 23, "tokens/total": 732840, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 10447} +{"epoch": 0.09375, "grad_norm": 1.8840023279190063, "learning_rate": 9.200000000000001e-05, "loss": 0.027712907642126083, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.0281, "step": 24, "tokens/total": 763176, "tokens/train_per_sec_per_gpu": 9.88, "tokens/trainable": 10899} +{"epoch": 0.09765625, "grad_norm": 2.0233547687530518, "learning_rate": 9.6e-05, "loss": 0.06482268124818802, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.06697, "step": 25, "tokens/total": 793464, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 11360} +{"epoch": 0.1015625, "grad_norm": 12.764789581298828, "learning_rate": 0.0001, "loss": 0.04993787407875061, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.05121, "step": 26, "tokens/total": 820920, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 11775} +{"epoch": 0.10546875, "grad_norm": 8.050917625427246, "learning_rate": 9.99990636831587e-05, "loss": 0.06578300148248672, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.06799, "step": 27, "tokens/total": 852400, "tokens/train_per_sec_per_gpu": 10.12, "tokens/trainable": 12242} +{"epoch": 0.109375, "grad_norm": 1.2999248504638672, "learning_rate": 9.999625477159879e-05, "loss": 0.049959227442741394, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.05123, "step": 28, "tokens/total": 887720, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 12696} +{"epoch": 0.11328125, "grad_norm": 5.602549076080322, "learning_rate": 9.999157338221051e-05, "loss": 0.023589886724948883, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.02387, "step": 29, "tokens/total": 919640, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 13153} +{"epoch": 0.1171875, "grad_norm": 0.5851481556892395, "learning_rate": 9.998501970980562e-05, "loss": 0.030104760080575943, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.03056, "step": 30, "tokens/total": 949736, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 13624} +{"epoch": 0.12109375, "grad_norm": 0.5269538164138794, "learning_rate": 9.997659402710915e-05, "loss": 0.02606319449841976, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.02641, "step": 31, "tokens/total": 983624, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 14064} +{"epoch": 0.125, "grad_norm": 0.6284741163253784, "learning_rate": 9.996629668474818e-05, "loss": 0.02990829385817051, "memory/device_reserved (GiB)": 66.67, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.03036, "step": 32, "tokens/total": 1017392, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 14565} +{"epoch": 0.12890625, "grad_norm": 0.8036213517189026, "learning_rate": 9.995412811123711e-05, "loss": 0.026363003998994827, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.02671, "step": 33, "tokens/total": 1049600, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 15005} +{"epoch": 0.1328125, "grad_norm": 0.4358982443809509, "learning_rate": 9.994008881295999e-05, "loss": 0.015864059329032898, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.01599, "step": 34, "tokens/total": 1082016, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 15459} +{"epoch": 0.13671875, "grad_norm": 0.7393144369125366, "learning_rate": 9.992417937414932e-05, "loss": 0.025405051186680794, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.02573, "step": 35, "tokens/total": 1113368, "tokens/train_per_sec_per_gpu": 10.15, "tokens/trainable": 15960} +{"epoch": 0.140625, "grad_norm": 0.7551636099815369, "learning_rate": 9.99064004568618e-05, "loss": 0.01802222803235054, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.84, "memory/max_allocated (GiB)": 64.84, "ppl": 1.01819, "step": 36, "tokens/total": 1146000, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 16428} +{"epoch": 0.14453125, "grad_norm": 0.6519080400466919, "learning_rate": 9.988675280095074e-05, "loss": 0.019854281097650528, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.27, "memory/max_allocated (GiB)": 64.27, "ppl": 1.02005, "step": 37, "tokens/total": 1176800, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 16884} +{"epoch": 0.1484375, "grad_norm": 2.018028974533081, "learning_rate": 9.986523722403528e-05, "loss": 0.03686889633536339, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.03756, "step": 38, "tokens/total": 1207232, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 17341} +{"epoch": 0.15234375, "grad_norm": 0.7356778979301453, "learning_rate": 9.984185462146642e-05, "loss": 0.015464391559362411, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.01558, "step": 39, "tokens/total": 1239504, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 17786} +{"epoch": 0.15625, "grad_norm": 0.6195996403694153, "learning_rate": 9.98166059662897e-05, "loss": 0.024697769433259964, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.76, "memory/max_allocated (GiB)": 65.76, "ppl": 1.02501, "step": 40, "tokens/total": 1272096, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 18262} +{"epoch": 0.16015625, "grad_norm": 0.3021087050437927, "learning_rate": 9.978949230920472e-05, "loss": 0.01137470081448555, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.01144, "step": 41, "tokens/total": 1304584, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 18716} +{"epoch": 0.1640625, "grad_norm": 1.0090785026550293, "learning_rate": 9.976051477852141e-05, "loss": 0.047017913311719894, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.04814, "step": 42, "tokens/total": 1337376, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 19157} +{"epoch": 0.16796875, "grad_norm": 0.8177807927131653, "learning_rate": 9.972967458011312e-05, "loss": 0.03983340412378311, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.04064, "step": 43, "tokens/total": 1369576, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 19647} +{"epoch": 0.171875, "grad_norm": 0.7400243282318115, "learning_rate": 9.96969729973664e-05, "loss": 0.02925274707376957, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.02968, "step": 44, "tokens/total": 1407336, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 20119} +{"epoch": 0.17578125, "grad_norm": 0.3941298723220825, "learning_rate": 9.966241139112754e-05, "loss": 0.020596910268068314, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.02081, "step": 45, "tokens/total": 1439392, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 20560} +{"epoch": 0.1796875, "grad_norm": 0.46469470858573914, "learning_rate": 9.96259911996461e-05, "loss": 0.015443782322108746, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01556, "step": 46, "tokens/total": 1471120, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 20992} +{"epoch": 0.18359375, "grad_norm": 0.7687447667121887, "learning_rate": 9.958771393851491e-05, "loss": 0.03327377885580063, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.03383, "step": 47, "tokens/total": 1502272, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 21441} +{"epoch": 0.1875, "grad_norm": 1.4158194065093994, "learning_rate": 9.954758120060702e-05, "loss": 0.030164893716573715, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.03062, "step": 48, "tokens/total": 1534296, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 21901} +{"epoch": 0.19140625, "grad_norm": 0.34912076592445374, "learning_rate": 9.950559465600948e-05, "loss": 0.006016354076564312, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00603, "step": 49, "tokens/total": 1566544, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 22341} +{"epoch": 0.1953125, "grad_norm": 3.0081307888031006, "learning_rate": 9.946175605195379e-05, "loss": 0.01300069596618414, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01309, "step": 50, "tokens/total": 1601760, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 22833} +{"epoch": 0.19921875, "grad_norm": 0.7136260271072388, "learning_rate": 9.941606721274322e-05, "loss": 0.013775100000202656, "memory/device_reserved (GiB)": 66.98, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.01387, "step": 51, "tokens/total": 1634312, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 23277} +{"epoch": 0.203125, "grad_norm": 1.946993350982666, "learning_rate": 9.936853003967685e-05, "loss": 0.037398193031549454, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.03811, "step": 52, "tokens/total": 1668232, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 23759} +{"epoch": 0.20703125, "grad_norm": 4.375123500823975, "learning_rate": 9.93191465109705e-05, "loss": 0.05614097788929939, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.05775, "step": 53, "tokens/total": 1701160, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 24193} +{"epoch": 0.2109375, "grad_norm": 1.3069405555725098, "learning_rate": 9.926791868167438e-05, "loss": 0.04767129570245743, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.04883, "step": 54, "tokens/total": 1735968, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 24619} +{"epoch": 0.21484375, "grad_norm": 1.739688515663147, "learning_rate": 9.921484868358753e-05, "loss": 0.03211739659309387, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.03264, "step": 55, "tokens/total": 1767624, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 25075} +{"epoch": 0.21875, "grad_norm": 0.390572190284729, "learning_rate": 9.915993872516924e-05, "loss": 0.0226382277905941, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.0229, "step": 56, "tokens/total": 1800536, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 25519} +{"epoch": 0.22265625, "grad_norm": 0.2646920680999756, "learning_rate": 9.9103191091447e-05, "loss": 0.008086375892162323, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00812, "step": 57, "tokens/total": 1830464, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 25966} +{"epoch": 0.2265625, "grad_norm": 1.1944547891616821, "learning_rate": 9.904460814392147e-05, "loss": 0.02106413245201111, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.02129, "step": 58, "tokens/total": 1867568, "tokens/train_per_sec_per_gpu": 6.95, "tokens/trainable": 26423} +{"epoch": 0.23046875, "grad_norm": 0.47500911355018616, "learning_rate": 9.898419232046825e-05, "loss": 0.012411735020577908, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01249, "step": 59, "tokens/total": 1902424, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 26934} +{"epoch": 0.234375, "grad_norm": 0.4158560335636139, "learning_rate": 9.892194613523633e-05, "loss": 0.011373201385140419, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01144, "step": 60, "tokens/total": 1937032, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 27393} +{"epoch": 0.23828125, "grad_norm": 0.41738975048065186, "learning_rate": 9.885787217854357e-05, "loss": 0.008045738562941551, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00808, "step": 61, "tokens/total": 1968960, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 27852} +{"epoch": 0.2421875, "grad_norm": 1.1671806573867798, "learning_rate": 9.879197311676887e-05, "loss": 0.020092889666557312, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.0203, "step": 62, "tokens/total": 2001688, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 28292} +{"epoch": 0.24609375, "grad_norm": 0.5855306386947632, "learning_rate": 9.872425169224113e-05, "loss": 0.02031031996011734, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.02052, "step": 63, "tokens/total": 2034160, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 28798} +{"epoch": 0.25, "grad_norm": 1.3142374753952026, "learning_rate": 9.865471072312528e-05, "loss": 0.024390827864408493, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.02469, "step": 64, "tokens/total": 2069552, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 29283} +{"epoch": 0.25390625, "grad_norm": 0.5775626301765442, "learning_rate": 9.858335310330492e-05, "loss": 0.005149744916707277, "memory/device_reserved (GiB)": 66.99, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00516, "step": 65, "tokens/total": 2099280, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 29745} +{"epoch": 0.2578125, "grad_norm": 0.8297920227050781, "learning_rate": 9.851018180226185e-05, "loss": 0.01403906662017107, "memory/device_reserved (GiB)": 66.17, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01414, "step": 66, "tokens/total": 2131280, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 30186} +{"epoch": 0.26171875, "grad_norm": 0.8434745073318481, "learning_rate": 9.843519986495259e-05, "loss": 0.016583099961280823, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.01672, "step": 67, "tokens/total": 2162232, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 30622} +{"epoch": 0.265625, "grad_norm": 0.4264957904815674, "learning_rate": 9.835841041168162e-05, "loss": 0.007735072169452906, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00777, "step": 68, "tokens/total": 2197080, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 31097} +{"epoch": 0.26953125, "grad_norm": 4.2112908363342285, "learning_rate": 9.82798166379715e-05, "loss": 0.0029910635203123093, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.003, "step": 69, "tokens/total": 2228208, "tokens/train_per_sec_per_gpu": 10.27, "tokens/trainable": 31595} +{"epoch": 0.2734375, "grad_norm": 0.3299919366836548, "learning_rate": 9.819942181443002e-05, "loss": 0.011904598213732243, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.01198, "step": 70, "tokens/total": 2260704, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 32036} +{"epoch": 0.27734375, "grad_norm": 2.3447859287261963, "learning_rate": 9.811722928661392e-05, "loss": 0.013866564258933067, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 63.2, "memory/max_allocated (GiB)": 63.2, "ppl": 1.01396, "step": 71, "tokens/total": 2288584, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 32428} +{"epoch": 0.28125, "grad_norm": 0.49227580428123474, "learning_rate": 9.803324247488975e-05, "loss": 0.008082525804638863, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00812, "step": 72, "tokens/total": 2320912, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 32880} +{"epoch": 0.28515625, "grad_norm": 0.4338599443435669, "learning_rate": 9.794746487429161e-05, "loss": 0.004099973943084478, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.04, "memory/max_allocated (GiB)": 64.04, "ppl": 1.00411, "step": 73, "tokens/total": 2352096, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 33323} +{"epoch": 0.2890625, "grad_norm": 0.34370172023773193, "learning_rate": 9.785990005437554e-05, "loss": 0.014912979677319527, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01502, "step": 74, "tokens/total": 2385328, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 33792} +{"epoch": 0.29296875, "grad_norm": 1.3955658674240112, "learning_rate": 9.777055165907117e-05, "loss": 0.021476464346051216, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.02171, "step": 75, "tokens/total": 2416416, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 34223} +{"epoch": 0.296875, "grad_norm": 0.9419927000999451, "learning_rate": 9.767942340652993e-05, "loss": 0.008971556089818478, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00901, "step": 76, "tokens/total": 2446192, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 34649} +{"epoch": 0.30078125, "grad_norm": 0.6740560531616211, "learning_rate": 9.758651908897035e-05, "loss": 0.01376230176538229, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01386, "step": 77, "tokens/total": 2478312, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 35094} +{"epoch": 0.3046875, "grad_norm": 1.0734854936599731, "learning_rate": 9.749184257252033e-05, "loss": 0.019695833325386047, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.01989, "step": 78, "tokens/total": 2509984, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 35534} +{"epoch": 0.30859375, "grad_norm": 1.1920695304870605, "learning_rate": 9.739539779705614e-05, "loss": 0.018084295094013214, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.01825, "step": 79, "tokens/total": 2546536, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 36029} +{"epoch": 0.3125, "grad_norm": 0.8595220446586609, "learning_rate": 9.729718877603861e-05, "loss": 0.018059296533465385, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.01822, "step": 80, "tokens/total": 2578048, "tokens/train_per_sec_per_gpu": 8.71, "tokens/trainable": 36469} +{"epoch": 0.31640625, "grad_norm": 0.9355340600013733, "learning_rate": 9.719721959634592e-05, "loss": 0.012827894650399685, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.01291, "step": 81, "tokens/total": 2609640, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 36906} +{"epoch": 0.3203125, "grad_norm": 0.9144654870033264, "learning_rate": 9.709549441810375e-05, "loss": 0.006988869048655033, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00701, "step": 82, "tokens/total": 2644408, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 37390} +{"epoch": 0.32421875, "grad_norm": 0.8151845335960388, "learning_rate": 9.699201747451195e-05, "loss": 0.01625499688088894, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.01639, "step": 83, "tokens/total": 2675576, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 37838} +{"epoch": 0.328125, "grad_norm": 0.3121548593044281, "learning_rate": 9.688679307166854e-05, "loss": 0.0034281155094504356, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00343, "step": 84, "tokens/total": 2706448, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 38310} +{"epoch": 0.33203125, "grad_norm": 0.35209834575653076, "learning_rate": 9.677982558839042e-05, "loss": 0.0074564614333212376, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00748, "step": 85, "tokens/total": 2738824, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 38789} +{"epoch": 0.3359375, "grad_norm": 0.5580830574035645, "learning_rate": 9.66711194760312e-05, "loss": 0.011741148307919502, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.01181, "step": 86, "tokens/total": 2775216, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 39277} +{"epoch": 0.33984375, "grad_norm": 1.1324764490127563, "learning_rate": 9.656067925829593e-05, "loss": 0.023797638714313507, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.02408, "step": 87, "tokens/total": 2804720, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 39737} +{"epoch": 0.34375, "grad_norm": 0.4348902404308319, "learning_rate": 9.644850953105288e-05, "loss": 0.007040842901915312, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.56, "memory/max_allocated (GiB)": 63.56, "ppl": 1.00707, "step": 88, "tokens/total": 2833944, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 40179} +{"epoch": 0.34765625, "grad_norm": 0.39318525791168213, "learning_rate": 9.633461496214225e-05, "loss": 0.009322211146354675, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00937, "step": 89, "tokens/total": 2865800, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 40649} +{"epoch": 0.3515625, "grad_norm": 0.8153624534606934, "learning_rate": 9.621900029118195e-05, "loss": 0.02693171799182892, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.0273, "step": 90, "tokens/total": 2894864, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 41080} +{"epoch": 0.35546875, "grad_norm": 16.538002014160156, "learning_rate": 9.610167032937036e-05, "loss": 0.02141588181257248, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.02165, "step": 91, "tokens/total": 2929216, "tokens/train_per_sec_per_gpu": 9.28, "tokens/trainable": 41599} +{"epoch": 0.359375, "grad_norm": 0.3943934142589569, "learning_rate": 9.598262995928611e-05, "loss": 0.012516479939222336, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.0126, "step": 92, "tokens/total": 2959280, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 42076} +{"epoch": 0.36328125, "grad_norm": 0.3677976131439209, "learning_rate": 9.586188413468492e-05, "loss": 0.00422469200566411, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00423, "step": 93, "tokens/total": 2993456, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 42522} +{"epoch": 0.3671875, "grad_norm": 0.3199220597743988, "learning_rate": 9.57394378802934e-05, "loss": 0.008183135651051998, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00822, "step": 94, "tokens/total": 3025752, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 42974} +{"epoch": 0.37109375, "grad_norm": 0.3127724528312683, "learning_rate": 9.56152962916e-05, "loss": 0.01157272607088089, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.55, "memory/max_allocated (GiB)": 63.55, "ppl": 1.01164, "step": 95, "tokens/total": 3054192, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 43380} +{"epoch": 0.375, "grad_norm": 0.27340808510780334, "learning_rate": 9.548946453464296e-05, "loss": 0.006594757549464703, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00662, "step": 96, "tokens/total": 3087160, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 43865} +{"epoch": 0.37890625, "grad_norm": 1.6368526220321655, "learning_rate": 9.53619478457953e-05, "loss": 0.012845495715737343, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01293, "step": 97, "tokens/total": 3121000, "tokens/train_per_sec_per_gpu": 7.04, "tokens/trainable": 44328} +{"epoch": 0.3828125, "grad_norm": 1.8226721286773682, "learning_rate": 9.523275153154695e-05, "loss": 0.008798260241746902, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00884, "step": 98, "tokens/total": 3152272, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 44778} +{"epoch": 0.38671875, "grad_norm": 1.11911141872406, "learning_rate": 9.51018809682839e-05, "loss": 0.03929878771305084, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.04008, "step": 99, "tokens/total": 3185480, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 45225} +{"epoch": 0.390625, "grad_norm": 0.19954290986061096, "learning_rate": 9.49693416020645e-05, "loss": 0.003198453690856695, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.0032, "step": 100, "tokens/total": 3219360, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 45678} +{"epoch": 0.39453125, "grad_norm": 0.13981932401657104, "learning_rate": 9.483513894839276e-05, "loss": 0.0027507643681019545, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00275, "step": 101, "tokens/total": 3251368, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 46125} +{"epoch": 0.3984375, "grad_norm": 0.37091538310050964, "learning_rate": 9.469927859198888e-05, "loss": 0.011167013086378574, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01123, "step": 102, "tokens/total": 3284184, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 46612} +{"epoch": 0.40234375, "grad_norm": 0.5129191279411316, "learning_rate": 9.456176618655689e-05, "loss": 0.011993291787803173, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.01207, "step": 103, "tokens/total": 3314752, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 47059} +{"epoch": 0.40625, "grad_norm": 1.9265928268432617, "learning_rate": 9.442260745454927e-05, "loss": 0.008276363834738731, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00831, "step": 104, "tokens/total": 3349736, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 47536} +{"epoch": 0.41015625, "grad_norm": 0.1530236154794693, "learning_rate": 9.428180818692884e-05, "loss": 0.00334378844127059, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00335, "step": 105, "tokens/total": 3384224, "tokens/train_per_sec_per_gpu": 7.33, "tokens/trainable": 48037} +{"epoch": 0.4140625, "grad_norm": 0.17238815128803253, "learning_rate": 9.413937424292791e-05, "loss": 0.002901434898376465, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00291, "step": 106, "tokens/total": 3420808, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 48491} +{"epoch": 0.41796875, "grad_norm": 0.5220876336097717, "learning_rate": 9.399531154980424e-05, "loss": 0.0131302485242486, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.01322, "step": 107, "tokens/total": 3453512, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 48940} +{"epoch": 0.421875, "grad_norm": 0.6111719608306885, "learning_rate": 9.384962610259455e-05, "loss": 0.02599518746137619, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.02634, "step": 108, "tokens/total": 3486192, "tokens/train_per_sec_per_gpu": 7.17, "tokens/trainable": 49389} +{"epoch": 0.42578125, "grad_norm": 0.3234666585922241, "learning_rate": 9.370232396386494e-05, "loss": 0.008281746879220009, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00832, "step": 109, "tokens/total": 3515384, "tokens/train_per_sec_per_gpu": 10.19, "tokens/trainable": 49838} +{"epoch": 0.4296875, "grad_norm": 0.849112331867218, "learning_rate": 9.355341126345868e-05, "loss": 0.019072934985160828, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01926, "step": 110, "tokens/total": 3548264, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 50310} +{"epoch": 0.43359375, "grad_norm": 0.2282838076353073, "learning_rate": 9.340289419824107e-05, "loss": 0.004122537560760975, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 63.85, "memory/max_allocated (GiB)": 63.85, "ppl": 1.00413, "step": 111, "tokens/total": 3579288, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 50755} +{"epoch": 0.4375, "grad_norm": 0.2899918854236603, "learning_rate": 9.325077903184159e-05, "loss": 0.006841475144028664, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00686, "step": 112, "tokens/total": 3611240, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 51213} +{"epoch": 0.44140625, "grad_norm": 0.17141295969486237, "learning_rate": 9.30970720943932e-05, "loss": 0.003435170743614435, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00344, "step": 113, "tokens/total": 3644136, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 51660} +{"epoch": 0.4453125, "grad_norm": 0.13625629246234894, "learning_rate": 9.2941779782269e-05, "loss": 0.0026718515437096357, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00268, "step": 114, "tokens/total": 3676368, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 52133} +{"epoch": 0.44921875, "grad_norm": 0.4116435945034027, "learning_rate": 9.278490855781596e-05, "loss": 0.005630859639495611, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00565, "step": 115, "tokens/total": 3707896, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 52580} +{"epoch": 0.453125, "grad_norm": 0.2633405327796936, "learning_rate": 9.262646494908604e-05, "loss": 0.001983687747269869, "memory/device_reserved (GiB)": 66.86, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00199, "step": 116, "tokens/total": 3742296, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 53033} +{"epoch": 0.45703125, "grad_norm": 0.37356042861938477, "learning_rate": 9.246645554956457e-05, "loss": 0.013466890901327133, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.01356, "step": 117, "tokens/total": 3776456, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 53517} +{"epoch": 0.4609375, "grad_norm": 0.30725130438804626, "learning_rate": 9.230488701789578e-05, "loss": 0.014489259570837021, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.01459, "step": 118, "tokens/total": 3811688, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 53967} +{"epoch": 0.46484375, "grad_norm": 0.9376926422119141, "learning_rate": 9.214176607760577e-05, "loss": 0.013633270747959614, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.01373, "step": 119, "tokens/total": 3847200, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 54430} +{"epoch": 0.46875, "grad_norm": 0.564542829990387, "learning_rate": 9.197709951682268e-05, "loss": 0.01583808846771717, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.46, "memory/max_allocated (GiB)": 64.46, "ppl": 1.01596, "step": 120, "tokens/total": 3877800, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 54896} +{"epoch": 0.47265625, "grad_norm": 0.4517430365085602, "learning_rate": 9.181089418799428e-05, "loss": 0.013420384377241135, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.01351, "step": 121, "tokens/total": 3910976, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 55379} +{"epoch": 0.4765625, "grad_norm": 0.31306084990501404, "learning_rate": 9.164315700760271e-05, "loss": 0.010362912900745869, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.01042, "step": 122, "tokens/total": 3943768, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 55803} +{"epoch": 0.48046875, "grad_norm": 2.190807342529297, "learning_rate": 9.147389495587671e-05, "loss": 0.01690347120165825, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.01705, "step": 123, "tokens/total": 3973816, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 56249} +{"epoch": 0.484375, "grad_norm": 0.4646100401878357, "learning_rate": 9.130311507650116e-05, "loss": 0.01058669202029705, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.01064, "step": 124, "tokens/total": 4004704, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 56713} +{"epoch": 0.48828125, "grad_norm": 6.9127397537231445, "learning_rate": 9.113082447632394e-05, "loss": 0.01706867106258869, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.01722, "step": 125, "tokens/total": 4036040, "tokens/train_per_sec_per_gpu": 10.35, "tokens/trainable": 57196} +{"epoch": 0.4921875, "grad_norm": 0.1655280590057373, "learning_rate": 9.09570303250602e-05, "loss": 0.005383522715419531, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.0054, "step": 126, "tokens/total": 4066792, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 57680} +{"epoch": 0.49609375, "grad_norm": 0.748624324798584, "learning_rate": 9.078173985499394e-05, "loss": 0.014752035960555077, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.01486, "step": 127, "tokens/total": 4096112, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 58110} +{"epoch": 0.5, "grad_norm": 0.20052289962768555, "learning_rate": 9.060496036067713e-05, "loss": 0.00801387894898653, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00805, "step": 128, "tokens/total": 4131472, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 58534} +{"epoch": 0.50390625, "grad_norm": 0.797584593296051, "learning_rate": 9.042669919862615e-05, "loss": 0.005048913415521383, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00506, "step": 129, "tokens/total": 4162912, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 58998} +{"epoch": 0.5078125, "grad_norm": 0.5323464870452881, "learning_rate": 9.024696378701557e-05, "loss": 0.015347853302955627, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.01547, "step": 130, "tokens/total": 4195768, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 59448} +{"epoch": 0.51171875, "grad_norm": 0.124326691031456, "learning_rate": 9.006576160536948e-05, "loss": 0.0028325701132416725, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00284, "step": 131, "tokens/total": 4227192, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 59906} +{"epoch": 0.515625, "grad_norm": 0.37672844529151917, "learning_rate": 8.988310019425035e-05, "loss": 0.0064270393922924995, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00645, "step": 132, "tokens/total": 4263304, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 60350} +{"epoch": 0.51953125, "grad_norm": 0.35212889313697815, "learning_rate": 8.969898715494506e-05, "loss": 0.007795417215675116, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00783, "step": 133, "tokens/total": 4297352, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 60831} +{"epoch": 0.5234375, "grad_norm": 0.6760237812995911, "learning_rate": 8.951343014914869e-05, "loss": 0.008335748687386513, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00837, "step": 134, "tokens/total": 4332152, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 61305} +{"epoch": 0.52734375, "grad_norm": 0.14980152249336243, "learning_rate": 8.932643689864568e-05, "loss": 0.0006914630066603422, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00069, "step": 135, "tokens/total": 4365272, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 61792} +{"epoch": 0.53125, "grad_norm": 0.5373712778091431, "learning_rate": 8.913801518498845e-05, "loss": 0.019900482147932053, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.0201, "step": 136, "tokens/total": 4399928, "tokens/train_per_sec_per_gpu": 6.82, "tokens/trainable": 62253} +{"epoch": 0.53515625, "grad_norm": 0.22936391830444336, "learning_rate": 8.894817284917364e-05, "loss": 0.0026475482154637575, "memory/device_reserved (GiB)": 66.9, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00265, "step": 137, "tokens/total": 4434336, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 62707} +{"epoch": 0.5390625, "grad_norm": 0.22659815847873688, "learning_rate": 8.875691779131569e-05, "loss": 0.0015277002239599824, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00153, "step": 138, "tokens/total": 4468872, "tokens/train_per_sec_per_gpu": 6.32, "tokens/trainable": 63145} +{"epoch": 0.54296875, "grad_norm": 0.6216554045677185, "learning_rate": 8.856425797031829e-05, "loss": 0.0027701924555003643, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00277, "step": 139, "tokens/total": 4502416, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 63587} +{"epoch": 0.546875, "grad_norm": 0.4229435622692108, "learning_rate": 8.837020140354295e-05, "loss": 0.002135233487933874, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00214, "step": 140, "tokens/total": 4536024, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 64052} +{"epoch": 0.55078125, "grad_norm": 1.6062222719192505, "learning_rate": 8.817475616647554e-05, "loss": 0.011027061380445957, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.01109, "step": 141, "tokens/total": 4568960, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 64526} +{"epoch": 0.5546875, "grad_norm": 0.5955772399902344, "learning_rate": 8.797793039239017e-05, "loss": 0.004900163970887661, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00491, "step": 142, "tokens/total": 4601656, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 64983} +{"epoch": 0.55859375, "grad_norm": 0.3640887141227722, "learning_rate": 8.777973227201069e-05, "loss": 0.002937349956482649, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00294, "step": 143, "tokens/total": 4636136, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 65492} +{"epoch": 0.5625, "grad_norm": 0.4131016731262207, "learning_rate": 8.758017005316988e-05, "loss": 0.011213385500013828, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.01128, "step": 144, "tokens/total": 4670160, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 65925} +{"epoch": 0.56640625, "grad_norm": 1.5096344947814941, "learning_rate": 8.737925204046629e-05, "loss": 0.04052946716547012, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.04136, "step": 145, "tokens/total": 4703792, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 66383} +{"epoch": 0.5703125, "grad_norm": 0.576728343963623, "learning_rate": 8.717698659491851e-05, "loss": 0.0013249937910586596, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00133, "step": 146, "tokens/total": 4737640, "tokens/train_per_sec_per_gpu": 6.93, "tokens/trainable": 66840} +{"epoch": 0.57421875, "grad_norm": 0.6794087290763855, "learning_rate": 8.697338213361735e-05, "loss": 0.01093764416873455, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.011, "step": 147, "tokens/total": 4770704, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 67297} +{"epoch": 0.578125, "grad_norm": 0.42933958768844604, "learning_rate": 8.676844712937552e-05, "loss": 0.004821670241653919, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00483, "step": 148, "tokens/total": 4807376, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 67769} +{"epoch": 0.58203125, "grad_norm": 0.9655292630195618, "learning_rate": 8.656219011037509e-05, "loss": 0.012348407879471779, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01242, "step": 149, "tokens/total": 4840728, "tokens/train_per_sec_per_gpu": 6.55, "tokens/trainable": 68257} +{"epoch": 0.5859375, "grad_norm": 0.28603750467300415, "learning_rate": 8.63546196598125e-05, "loss": 0.0034720380790531635, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00348, "step": 150, "tokens/total": 4873880, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 68706} +{"epoch": 0.58984375, "grad_norm": 0.2282860428094864, "learning_rate": 8.614574441554145e-05, "loss": 0.007142094895243645, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00717, "step": 151, "tokens/total": 4904512, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 69131} +{"epoch": 0.59375, "grad_norm": 0.32434967160224915, "learning_rate": 8.593557306971349e-05, "loss": 0.007278351113200188, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.0073, "step": 152, "tokens/total": 4937048, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 69586} +{"epoch": 0.59765625, "grad_norm": 1.027780532836914, "learning_rate": 8.572411436841618e-05, "loss": 0.007205259520560503, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00723, "step": 153, "tokens/total": 4968224, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 70061} +{"epoch": 0.6015625, "grad_norm": 0.15130481123924255, "learning_rate": 8.551137711130922e-05, "loss": 0.0033639175817370415, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00337, "step": 154, "tokens/total": 4997272, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 70467} +{"epoch": 0.60546875, "grad_norm": 0.2739264965057373, "learning_rate": 8.529737015125824e-05, "loss": 0.008057614788413048, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00809, "step": 155, "tokens/total": 5031072, "tokens/train_per_sec_per_gpu": 8.14, "tokens/trainable": 70933} +{"epoch": 0.609375, "grad_norm": 0.5382895469665527, "learning_rate": 8.508210239396639e-05, "loss": 0.02031658962368965, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.02052, "step": 156, "tokens/total": 5063808, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 71382} +{"epoch": 0.61328125, "grad_norm": 0.37698808312416077, "learning_rate": 8.486558279760375e-05, "loss": 0.005877090618014336, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.18, "memory/max_allocated (GiB)": 63.18, "ppl": 1.00589, "step": 157, "tokens/total": 5094032, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 71808} +{"epoch": 0.6171875, "grad_norm": 0.4238322675228119, "learning_rate": 8.464782037243449e-05, "loss": 0.00385114224627614, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00386, "step": 158, "tokens/total": 5125664, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 72249} +{"epoch": 0.62109375, "grad_norm": 0.262590616941452, "learning_rate": 8.442882418044202e-05, "loss": 0.00556769035756588, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00558, "step": 159, "tokens/total": 5156952, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 72726} +{"epoch": 0.625, "grad_norm": 1.327434778213501, "learning_rate": 8.420860333495179e-05, "loss": 0.011830498464405537, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.0119, "step": 160, "tokens/total": 5187120, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 73148} +{"epoch": 0.62890625, "grad_norm": 0.5036587715148926, "learning_rate": 8.398716700025208e-05, "loss": 0.0016751468647271395, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00168, "step": 161, "tokens/total": 5219080, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 73600} +{"epoch": 0.6328125, "grad_norm": 0.18569380044937134, "learning_rate": 8.376452439121266e-05, "loss": 0.0015467156190425158, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.27, "memory/max_allocated (GiB)": 63.27, "ppl": 1.00155, "step": 162, "tokens/total": 5248680, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 74068} +{"epoch": 0.63671875, "grad_norm": 2.5078892707824707, "learning_rate": 8.354068477290124e-05, "loss": 0.010038130916655064, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01009, "step": 163, "tokens/total": 5280104, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 74527} +{"epoch": 0.640625, "grad_norm": 0.21470904350280762, "learning_rate": 8.331565746019807e-05, "loss": 0.004275599028915167, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.76, "memory/max_allocated (GiB)": 63.76, "ppl": 1.00428, "step": 164, "tokens/total": 5311328, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 74992} +{"epoch": 0.64453125, "grad_norm": 2.952812433242798, "learning_rate": 8.308945181740812e-05, "loss": 0.019097033888101578, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.01928, "step": 165, "tokens/total": 5343960, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 75442} +{"epoch": 0.6484375, "grad_norm": 0.12322814762592316, "learning_rate": 8.286207725787153e-05, "loss": 0.0017624367028474808, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00176, "step": 166, "tokens/total": 5377312, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 75887} +{"epoch": 0.65234375, "grad_norm": 1.3447128534317017, "learning_rate": 8.263354324357182e-05, "loss": 0.02560277096927166, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.02593, "step": 167, "tokens/total": 5409824, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 76331} +{"epoch": 0.65625, "grad_norm": 0.42238613963127136, "learning_rate": 8.240385928474219e-05, "loss": 0.007391937542706728, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00742, "step": 168, "tokens/total": 5440904, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 76745} +{"epoch": 0.66015625, "grad_norm": 0.010034398175776005, "learning_rate": 8.217303493946967e-05, "loss": 8.361894288100302e-05, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00008, "step": 169, "tokens/total": 5474920, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 77201} +{"epoch": 0.6640625, "grad_norm": 0.3822285830974579, "learning_rate": 8.194107981329746e-05, "loss": 0.013912045396864414, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.01401, "step": 170, "tokens/total": 5511856, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 77655} +{"epoch": 0.66796875, "grad_norm": 0.03090502880513668, "learning_rate": 8.170800355882518e-05, "loss": 0.00019900646293535829, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.0002, "step": 171, "tokens/total": 5544104, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 78122} +{"epoch": 0.671875, "grad_norm": 1.0779814720153809, "learning_rate": 8.147381587530713e-05, "loss": 0.02174565941095352, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.02198, "step": 172, "tokens/total": 5577344, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 78576} +{"epoch": 0.67578125, "grad_norm": 0.4638008177280426, "learning_rate": 8.123852650824877e-05, "loss": 0.007290906272828579, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00732, "step": 173, "tokens/total": 5610024, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 79059} +{"epoch": 0.6796875, "grad_norm": 0.1511761099100113, "learning_rate": 8.100214524900103e-05, "loss": 0.0016447969246655703, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00165, "step": 174, "tokens/total": 5643184, "tokens/train_per_sec_per_gpu": 7.16, "tokens/trainable": 79498} +{"epoch": 0.68359375, "grad_norm": 1.6470232009887695, "learning_rate": 8.076468193435301e-05, "loss": 0.008168199099600315, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.0082, "step": 175, "tokens/total": 5672824, "tokens/train_per_sec_per_gpu": 8.58, "tokens/trainable": 79922} +{"epoch": 0.6875, "grad_norm": 0.4255550503730774, "learning_rate": 8.052614644612253e-05, "loss": 0.010392798110842705, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.01045, "step": 176, "tokens/total": 5705576, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 80383} +{"epoch": 0.69140625, "grad_norm": 0.09934861958026886, "learning_rate": 8.028654871074489e-05, "loss": 0.0014941239496693015, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0015, "step": 177, "tokens/total": 5737840, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 80824} +{"epoch": 0.6953125, "grad_norm": 0.21127626299858093, "learning_rate": 8.004589869885986e-05, "loss": 0.003556588664650917, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00356, "step": 178, "tokens/total": 5767552, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 81243} +{"epoch": 0.69921875, "grad_norm": 0.4159478545188904, "learning_rate": 7.980420642489674e-05, "loss": 0.015162664465606213, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.01528, "step": 179, "tokens/total": 5802512, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 81716} +{"epoch": 0.703125, "grad_norm": 0.07373679429292679, "learning_rate": 7.95614819466576e-05, "loss": 0.0016753195086494088, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00168, "step": 180, "tokens/total": 5834984, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 82181} +{"epoch": 0.70703125, "grad_norm": 0.6532506346702576, "learning_rate": 7.931773536489872e-05, "loss": 0.0028208461590111256, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00282, "step": 181, "tokens/total": 5867336, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 82626} +{"epoch": 0.7109375, "grad_norm": 0.10313202440738678, "learning_rate": 7.907297682291035e-05, "loss": 0.002214907668530941, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00222, "step": 182, "tokens/total": 5899232, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 83089} +{"epoch": 0.71484375, "grad_norm": 0.1551458090543747, "learning_rate": 7.882721650609442e-05, "loss": 0.004041478503495455, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00405, "step": 183, "tokens/total": 5934816, "tokens/train_per_sec_per_gpu": 6.64, "tokens/trainable": 83590} +{"epoch": 0.71875, "grad_norm": 0.254025936126709, "learning_rate": 7.85804646415409e-05, "loss": 0.006662965286523104, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.00669, "step": 184, "tokens/total": 5968120, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 84046} +{"epoch": 0.72265625, "grad_norm": 0.20878662168979645, "learning_rate": 7.833273149760207e-05, "loss": 0.0021088337525725365, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00211, "step": 185, "tokens/total": 5998600, "tokens/train_per_sec_per_gpu": 9.73, "tokens/trainable": 84517} +{"epoch": 0.7265625, "grad_norm": 0.397853285074234, "learning_rate": 7.808402738346527e-05, "loss": 0.008527304977178574, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00856, "step": 186, "tokens/total": 6033832, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 84974} +{"epoch": 0.73046875, "grad_norm": 0.1451658010482788, "learning_rate": 7.783436264872382e-05, "loss": 0.0014436299679800868, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00144, "step": 187, "tokens/total": 6068192, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 85460} +{"epoch": 0.734375, "grad_norm": 0.43793296813964844, "learning_rate": 7.758374768294647e-05, "loss": 0.01613527163863182, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.01627, "step": 188, "tokens/total": 6101464, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 85939} +{"epoch": 0.73828125, "grad_norm": 0.4431740343570709, "learning_rate": 7.733219291524489e-05, "loss": 0.00582929328083992, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00585, "step": 189, "tokens/total": 6133008, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 86377} +{"epoch": 0.7421875, "grad_norm": 0.06412375718355179, "learning_rate": 7.707970881383977e-05, "loss": 0.0008838659850880504, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.00088, "step": 190, "tokens/total": 6163048, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 86834} +{"epoch": 0.74609375, "grad_norm": 0.22799764573574066, "learning_rate": 7.682630588562518e-05, "loss": 0.00586107699200511, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00588, "step": 191, "tokens/total": 6196072, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 87265} +{"epoch": 0.75, "grad_norm": 0.6686044931411743, "learning_rate": 7.657199467573129e-05, "loss": 0.0031958180479705334, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.0032, "step": 192, "tokens/total": 6227312, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 87747} +{"epoch": 0.75390625, "grad_norm": 0.034844107925891876, "learning_rate": 7.631678576708561e-05, "loss": 0.00043586132233031094, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00044, "step": 193, "tokens/total": 6260240, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 88239} +{"epoch": 0.7578125, "grad_norm": 0.022614067420363426, "learning_rate": 7.606068977997255e-05, "loss": 0.00028036098228767514, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00028, "step": 194, "tokens/total": 6294280, "tokens/train_per_sec_per_gpu": 9.2, "tokens/trainable": 88718} +{"epoch": 0.76171875, "grad_norm": 0.3971022069454193, "learning_rate": 7.580371737159148e-05, "loss": 0.007029606495052576, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00705, "step": 195, "tokens/total": 6323368, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 89129} +{"epoch": 0.765625, "grad_norm": 0.017747381702065468, "learning_rate": 7.554587923561324e-05, "loss": 0.00020561451674439013, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00021, "step": 196, "tokens/total": 6354960, "tokens/train_per_sec_per_gpu": 8.94, "tokens/trainable": 89567} +{"epoch": 0.76953125, "grad_norm": 0.2634125053882599, "learning_rate": 7.528718610173511e-05, "loss": 0.003095556516200304, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.0031, "step": 197, "tokens/total": 6387208, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 89988} +{"epoch": 0.7734375, "grad_norm": 0.10673216730356216, "learning_rate": 7.502764873523431e-05, "loss": 0.0004107675631530583, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00041, "step": 198, "tokens/total": 6418720, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 90434} +{"epoch": 0.77734375, "grad_norm": 0.10897145420312881, "learning_rate": 7.476727793652011e-05, "loss": 0.0012884290190413594, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00129, "step": 199, "tokens/total": 6449184, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 90913} +{"epoch": 0.78125, "grad_norm": 0.6502819061279297, "learning_rate": 7.450608454068415e-05, "loss": 0.008770525455474854, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00881, "step": 200, "tokens/total": 6481824, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 91355} +{"epoch": 0.78515625, "grad_norm": 0.3737095296382904, "learning_rate": 7.424407941704987e-05, "loss": 0.006353884469717741, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.00637, "step": 201, "tokens/total": 6515896, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 91856} +{"epoch": 0.7890625, "grad_norm": 0.1599770039319992, "learning_rate": 7.398127346871986e-05, "loss": 0.002622312866151333, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00263, "step": 202, "tokens/total": 6548784, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 92311} +{"epoch": 0.79296875, "grad_norm": 0.05094454437494278, "learning_rate": 7.371767763212238e-05, "loss": 0.000297654332825914, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.0003, "step": 203, "tokens/total": 6580256, "tokens/train_per_sec_per_gpu": 9.91, "tokens/trainable": 92764} +{"epoch": 0.796875, "grad_norm": 0.09096265584230423, "learning_rate": 7.345330287655617e-05, "loss": 0.001690498786047101, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00169, "step": 204, "tokens/total": 6613976, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 93227} +{"epoch": 0.80078125, "grad_norm": 0.5904133319854736, "learning_rate": 7.31881602037339e-05, "loss": 0.01024983823299408, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.0103, "step": 205, "tokens/total": 6644552, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 93675} +{"epoch": 0.8046875, "grad_norm": 0.3487303853034973, "learning_rate": 7.29222606473245e-05, "loss": 0.007562803570181131, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00759, "step": 206, "tokens/total": 6675056, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 94120} +{"epoch": 0.80859375, "grad_norm": 0.24715393781661987, "learning_rate": 7.265561527249383e-05, "loss": 0.003111595753580332, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00312, "step": 207, "tokens/total": 6706624, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 94557} +{"epoch": 0.8125, "grad_norm": 0.042390353977680206, "learning_rate": 7.238823517544436e-05, "loss": 0.00037609264836646616, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00038, "step": 208, "tokens/total": 6741008, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 95035} +{"epoch": 0.81640625, "grad_norm": 0.4766272008419037, "learning_rate": 7.212013148295333e-05, "loss": 0.006320840213447809, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00634, "step": 209, "tokens/total": 6774872, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 95517} +{"epoch": 0.8203125, "grad_norm": 0.08458428084850311, "learning_rate": 7.185131535190975e-05, "loss": 0.000754803535528481, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00076, "step": 210, "tokens/total": 6805816, "tokens/train_per_sec_per_gpu": 9.48, "tokens/trainable": 96026} +{"epoch": 0.82421875, "grad_norm": 0.20057615637779236, "learning_rate": 7.158179796885005e-05, "loss": 0.0027513152454048395, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00276, "step": 211, "tokens/total": 6837592, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 96477} +{"epoch": 0.828125, "grad_norm": 0.16387026011943817, "learning_rate": 7.131159054949273e-05, "loss": 0.0017944187857210636, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.0018, "step": 212, "tokens/total": 6874384, "tokens/train_per_sec_per_gpu": 6.98, "tokens/trainable": 96951} +{"epoch": 0.83203125, "grad_norm": 0.2513733208179474, "learning_rate": 7.104070433827139e-05, "loss": 0.004958381410688162, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00497, "step": 213, "tokens/total": 6905872, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 97350} +{"epoch": 0.8359375, "grad_norm": 0.1413390189409256, "learning_rate": 7.076915060786705e-05, "loss": 0.00034872221294790506, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.25, "memory/max_allocated (GiB)": 65.25, "ppl": 1.00035, "step": 214, "tokens/total": 6937984, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 97792} +{"epoch": 0.83984375, "grad_norm": 0.6960380673408508, "learning_rate": 7.049694065873882e-05, "loss": 0.011198672465980053, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.01126, "step": 215, "tokens/total": 6970704, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 98257} +{"epoch": 0.84375, "grad_norm": 0.21141399443149567, "learning_rate": 7.022408581865382e-05, "loss": 0.002856910927221179, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00286, "step": 216, "tokens/total": 7004616, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 98731} +{"epoch": 0.84765625, "grad_norm": 0.1286625862121582, "learning_rate": 6.99505974422157e-05, "loss": 0.00017558250692673028, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00018, "step": 217, "tokens/total": 7036880, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 99212} +{"epoch": 0.8515625, "grad_norm": 0.3235255479812622, "learning_rate": 6.967648691039213e-05, "loss": 0.0015368455788120627, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00154, "step": 218, "tokens/total": 7068208, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 99654} +{"epoch": 0.85546875, "grad_norm": 0.046584147959947586, "learning_rate": 6.940176563004123e-05, "loss": 0.0004132200265303254, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00041, "step": 219, "tokens/total": 7098112, "tokens/train_per_sec_per_gpu": 7.47, "tokens/trainable": 100086} +{"epoch": 0.859375, "grad_norm": 0.08337733149528503, "learning_rate": 6.912644503343682e-05, "loss": 0.0009645131067372859, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00096, "step": 220, "tokens/total": 7132432, "tokens/train_per_sec_per_gpu": 7.04, "tokens/trainable": 100524} +{"epoch": 0.86328125, "grad_norm": 0.1764722764492035, "learning_rate": 6.885053657779273e-05, "loss": 0.0010232061613351107, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.9, "memory/max_allocated (GiB)": 63.9, "ppl": 1.00102, "step": 221, "tokens/total": 7164016, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 100996} +{"epoch": 0.8671875, "grad_norm": 0.09556639939546585, "learning_rate": 6.857405174478604e-05, "loss": 0.0005756462924182415, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00058, "step": 222, "tokens/total": 7198344, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 101449} +{"epoch": 0.87109375, "grad_norm": 0.09896855056285858, "learning_rate": 6.82970020400792e-05, "loss": 0.0007045047241263092, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.0007, "step": 223, "tokens/total": 7232192, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 101905} +{"epoch": 0.875, "grad_norm": 1.127111554145813, "learning_rate": 6.801939899284132e-05, "loss": 0.026524413377046585, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.02688, "step": 224, "tokens/total": 7266984, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 102411} +{"epoch": 0.87890625, "grad_norm": 0.09117089211940765, "learning_rate": 6.774125415526827e-05, "loss": 0.0005766822723671794, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00058, "step": 225, "tokens/total": 7299512, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 102882} +{"epoch": 0.8828125, "grad_norm": 0.018808528780937195, "learning_rate": 6.746257910210214e-05, "loss": 0.00014039759116712958, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.22, "memory/max_allocated (GiB)": 64.22, "ppl": 1.00014, "step": 226, "tokens/total": 7331120, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 103332} +{"epoch": 0.88671875, "grad_norm": 0.06918516010046005, "learning_rate": 6.718338543014937e-05, "loss": 0.0005644048796966672, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 63.31, "memory/max_allocated (GiB)": 63.31, "ppl": 1.00056, "step": 227, "tokens/total": 7360504, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 103790} +{"epoch": 0.890625, "grad_norm": 0.3621213734149933, "learning_rate": 6.69036847577983e-05, "loss": 0.003520084312185645, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00353, "step": 228, "tokens/total": 7390760, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 104246} +{"epoch": 0.89453125, "grad_norm": 0.0971241295337677, "learning_rate": 6.662348872453553e-05, "loss": 0.0010903524234890938, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00109, "step": 229, "tokens/total": 7423352, "tokens/train_per_sec_per_gpu": 10.33, "tokens/trainable": 104707} +{"epoch": 0.8984375, "grad_norm": 0.04709690064191818, "learning_rate": 6.63428089904618e-05, "loss": 0.00043716252548620105, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00044, "step": 230, "tokens/total": 7457152, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 105167} +{"epoch": 0.90234375, "grad_norm": 0.2998490035533905, "learning_rate": 6.60616572358065e-05, "loss": 0.009793579578399658, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00984, "step": 231, "tokens/total": 7488648, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 105576} +{"epoch": 0.90625, "grad_norm": 0.5656829476356506, "learning_rate": 6.578004516044172e-05, "loss": 0.0031803001184016466, "memory/device_reserved (GiB)": 66.94, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00319, "step": 232, "tokens/total": 7522160, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 106043} +{"epoch": 0.91015625, "grad_norm": 0.18366019427776337, "learning_rate": 6.549798448339548e-05, "loss": 0.0012337003136053681, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00123, "step": 233, "tokens/total": 7557336, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 106506} +{"epoch": 0.9140625, "grad_norm": 0.4228447675704956, "learning_rate": 6.521548694236384e-05, "loss": 0.005336962640285492, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00535, "step": 234, "tokens/total": 7590432, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 106951} +{"epoch": 0.91796875, "grad_norm": 0.43508797883987427, "learning_rate": 6.493256429322259e-05, "loss": 0.005276440177112818, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00529, "step": 235, "tokens/total": 7623240, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 107382} +{"epoch": 0.921875, "grad_norm": 0.798695981502533, "learning_rate": 6.464922830953799e-05, "loss": 0.012470560148358345, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.01255, "step": 236, "tokens/total": 7657968, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 107814} +{"epoch": 0.92578125, "grad_norm": 0.4022412598133087, "learning_rate": 6.436549078207688e-05, "loss": 0.006119300611317158, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00614, "step": 237, "tokens/total": 7689320, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 108274} +{"epoch": 0.9296875, "grad_norm": 7.682639122009277, "learning_rate": 6.408136351831592e-05, "loss": 0.017582911998033524, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.01774, "step": 238, "tokens/total": 7721568, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 108714} +{"epoch": 0.93359375, "grad_norm": 0.18321391940116882, "learning_rate": 6.379685834195036e-05, "loss": 0.001824679085984826, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00183, "step": 239, "tokens/total": 7756464, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 109220} +{"epoch": 0.9375, "grad_norm": 0.11705993860960007, "learning_rate": 6.351198709240186e-05, "loss": 0.0011023450642824173, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.0011, "step": 240, "tokens/total": 7791616, "tokens/train_per_sec_per_gpu": 6.89, "tokens/trainable": 109672} +{"epoch": 0.94140625, "grad_norm": 0.6344584822654724, "learning_rate": 6.32267616243259e-05, "loss": 0.009471931494772434, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00952, "step": 241, "tokens/total": 7823520, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 110135} +{"epoch": 0.9453125, "grad_norm": 0.17279121279716492, "learning_rate": 6.294119380711849e-05, "loss": 0.0007463719230145216, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00075, "step": 242, "tokens/total": 7854424, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 110563} +{"epoch": 0.94921875, "grad_norm": 0.548043966293335, "learning_rate": 6.265529552442209e-05, "loss": 0.015023954212665558, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.01514, "step": 243, "tokens/total": 7887160, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 111049} +{"epoch": 0.953125, "grad_norm": 0.6527654528617859, "learning_rate": 6.236907867363127e-05, "loss": 0.003203944070264697, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00321, "step": 244, "tokens/total": 7918736, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 111495} +{"epoch": 0.95703125, "grad_norm": 0.4032880663871765, "learning_rate": 6.208255516539749e-05, "loss": 0.002687630709260702, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00269, "step": 245, "tokens/total": 7949448, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 111968} +{"epoch": 0.9609375, "grad_norm": 0.0798131600022316, "learning_rate": 6.179573692313344e-05, "loss": 0.0010315875988453627, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.53, "memory/max_allocated (GiB)": 65.53, "ppl": 1.00103, "step": 246, "tokens/total": 7984680, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 112416} +{"epoch": 0.96484375, "grad_norm": 0.024264952167868614, "learning_rate": 6.150863588251694e-05, "loss": 0.0003885666956193745, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00039, "step": 247, "tokens/total": 8019816, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 112882} +{"epoch": 0.96875, "grad_norm": 0.15513674914836884, "learning_rate": 6.122126399099419e-05, "loss": 0.0009068147628568113, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00091, "step": 248, "tokens/total": 8056272, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 113390} +{"epoch": 0.97265625, "grad_norm": 0.7365327477455139, "learning_rate": 6.0933633207282615e-05, "loss": 0.01469306368380785, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.0148, "step": 249, "tokens/total": 8090224, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 113904} +{"epoch": 0.9765625, "grad_norm": 0.34989380836486816, "learning_rate": 6.064575550087316e-05, "loss": 0.0037688342854380608, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00378, "step": 250, "tokens/total": 8124528, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 114351} +{"epoch": 0.98046875, "grad_norm": 0.3068847060203552, "learning_rate": 6.0357642851532245e-05, "loss": 0.004798903129994869, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00481, "step": 251, "tokens/total": 8159688, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 114842} +{"epoch": 0.984375, "grad_norm": 0.19826097786426544, "learning_rate": 6.0069307248803294e-05, "loss": 0.0023650878574699163, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00237, "step": 252, "tokens/total": 8191800, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 115263} +{"epoch": 0.98828125, "grad_norm": 0.06879586726427078, "learning_rate": 5.9780760691507635e-05, "loss": 0.0010387962684035301, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00104, "step": 253, "tokens/total": 8221752, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 115703} +{"epoch": 0.9921875, "grad_norm": 0.33894190192222595, "learning_rate": 5.9492015187245334e-05, "loss": 0.007359566632658243, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00739, "step": 254, "tokens/total": 8255648, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 116157} +{"epoch": 0.99609375, "grad_norm": 0.3114739954471588, "learning_rate": 5.920308275189541e-05, "loss": 0.0010491556022316217, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00105, "step": 255, "tokens/total": 8286664, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 116567} +{"epoch": 1.0, "grad_norm": 0.024496274068951607, "learning_rate": 5.8913975409115874e-05, "loss": 0.0003761978878173977, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.00038, "step": 256, "tokens/total": 8318368, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 117030} +{"epoch": 1.00390625, "grad_norm": 0.040524374693632126, "learning_rate": 5.8624705189843395e-05, "loss": 0.0005960703711025417, "memory/device_reserved (GiB)": 66.95, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.0006, "step": 257, "tokens/total": 8353336, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 117517} +{"epoch": 1.0078125, "grad_norm": 0.28462281823158264, "learning_rate": 5.833528413179249e-05, "loss": 0.0010965607361868024, "memory/device_reserved (GiB)": 66.77, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.0011, "step": 258, "tokens/total": 8389112, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 118015} +{"epoch": 1.01171875, "grad_norm": 0.0831170603632927, "learning_rate": 5.80457242789548e-05, "loss": 0.0007413031999021769, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00074, "step": 259, "tokens/total": 8426000, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 118520} +{"epoch": 1.015625, "grad_norm": 0.01733480393886566, "learning_rate": 5.77560376810977e-05, "loss": 0.00025409896625205874, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.09, "memory/max_allocated (GiB)": 65.09, "ppl": 1.00025, "step": 260, "tokens/total": 8458568, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 118992} +{"epoch": 1.01953125, "grad_norm": 0.007027055136859417, "learning_rate": 5.7466236393263005e-05, "loss": 0.00014681807078886777, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00015, "step": 261, "tokens/total": 8490840, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 119438} +{"epoch": 1.0234375, "grad_norm": 0.32771310210227966, "learning_rate": 5.717633247526522e-05, "loss": 0.0019925471860915422, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00199, "step": 262, "tokens/total": 8521792, "tokens/train_per_sec_per_gpu": 8.42, "tokens/trainable": 119881} +{"epoch": 1.02734375, "grad_norm": 0.587576687335968, "learning_rate": 5.688633799118971e-05, "loss": 0.004744101781398058, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00476, "step": 263, "tokens/total": 8551336, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 120285} +{"epoch": 1.03125, "grad_norm": 0.03821064159274101, "learning_rate": 5.659626500889066e-05, "loss": 0.0006642700755037367, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00066, "step": 264, "tokens/total": 8579872, "tokens/train_per_sec_per_gpu": 6.91, "tokens/trainable": 120755} +{"epoch": 1.03515625, "grad_norm": 0.057417791336774826, "learning_rate": 5.6306125599488905e-05, "loss": 0.00045051166671328247, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.00045, "step": 265, "tokens/total": 8608920, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 121194} +{"epoch": 1.0390625, "grad_norm": 0.01985807716846466, "learning_rate": 5.601593183686955e-05, "loss": 0.00028357835253700614, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00028, "step": 266, "tokens/total": 8643600, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 121670} +{"epoch": 1.04296875, "grad_norm": 0.55670565366745, "learning_rate": 5.572569579717961e-05, "loss": 0.006724691949784756, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.00675, "step": 267, "tokens/total": 8676032, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 122142} +{"epoch": 1.046875, "grad_norm": 0.0059697930701076984, "learning_rate": 5.543542955832538e-05, "loss": 0.00011619695578701794, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00012, "step": 268, "tokens/total": 8713240, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 122585} +{"epoch": 1.05078125, "grad_norm": 0.1520485281944275, "learning_rate": 5.514514519946986e-05, "loss": 0.0024185229558497667, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00242, "step": 269, "tokens/total": 8751664, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 123091} +{"epoch": 1.0546875, "grad_norm": 0.02889794111251831, "learning_rate": 5.485485480053015e-05, "loss": 0.00012900825822725892, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00013, "step": 270, "tokens/total": 8786112, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 123505} +{"epoch": 1.05859375, "grad_norm": 0.012474361807107925, "learning_rate": 5.4564570441674645e-05, "loss": 0.0001498586789239198, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00015, "step": 271, "tokens/total": 8816920, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 123953} +{"epoch": 1.0625, "grad_norm": 0.09767942130565643, "learning_rate": 5.42743042028204e-05, "loss": 0.0011392018059268594, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00114, "step": 272, "tokens/total": 8851408, "tokens/train_per_sec_per_gpu": 6.69, "tokens/trainable": 124400} +{"epoch": 1.06640625, "grad_norm": 0.40830445289611816, "learning_rate": 5.3984068163130464e-05, "loss": 0.008229474537074566, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00826, "step": 273, "tokens/total": 8883768, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 124870} +{"epoch": 1.0703125, "grad_norm": 0.3297126591205597, "learning_rate": 5.369387440051111e-05, "loss": 0.0030700962524861097, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00307, "step": 274, "tokens/total": 8916200, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 125333} +{"epoch": 1.07421875, "grad_norm": 0.6373673677444458, "learning_rate": 5.340373499110935e-05, "loss": 0.00033235017326660454, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00033, "step": 275, "tokens/total": 8949920, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 125834} +{"epoch": 1.078125, "grad_norm": 0.3388592600822449, "learning_rate": 5.3113662008810304e-05, "loss": 0.0019018141319975257, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.0019, "step": 276, "tokens/total": 8986448, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 126316} +{"epoch": 1.08203125, "grad_norm": 0.5497495532035828, "learning_rate": 5.282366752473479e-05, "loss": 0.006108652800321579, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00613, "step": 277, "tokens/total": 9020600, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 126798} +{"epoch": 1.0859375, "grad_norm": 0.46269336342811584, "learning_rate": 5.2533763606737005e-05, "loss": 0.00692673958837986, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00695, "step": 278, "tokens/total": 9056040, "tokens/train_per_sec_per_gpu": 6.78, "tokens/trainable": 127223} +{"epoch": 1.08984375, "grad_norm": 0.33884018659591675, "learning_rate": 5.224396231890232e-05, "loss": 0.0035380651243031025, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00354, "step": 279, "tokens/total": 9087072, "tokens/train_per_sec_per_gpu": 9.56, "tokens/trainable": 127672} +{"epoch": 1.09375, "grad_norm": 0.135338693857193, "learning_rate": 5.195427572104522e-05, "loss": 0.001305272220633924, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00131, "step": 280, "tokens/total": 9118376, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 128116} +{"epoch": 1.09765625, "grad_norm": 0.145613893866539, "learning_rate": 5.166471586820751e-05, "loss": 0.001267931773327291, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00127, "step": 281, "tokens/total": 9152704, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 128589} +{"epoch": 1.1015625, "grad_norm": 0.35938069224357605, "learning_rate": 5.1375294810156615e-05, "loss": 0.0030491536017507315, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00305, "step": 282, "tokens/total": 9184448, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 129036} +{"epoch": 1.10546875, "grad_norm": 0.23993287980556488, "learning_rate": 5.1086024590884144e-05, "loss": 0.002231497084721923, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00223, "step": 283, "tokens/total": 9217392, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 129485} +{"epoch": 1.109375, "grad_norm": 0.20765377581119537, "learning_rate": 5.079691724810461e-05, "loss": 0.00043618539348244667, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00044, "step": 284, "tokens/total": 9252016, "tokens/train_per_sec_per_gpu": 6.99, "tokens/trainable": 129920} +{"epoch": 1.11328125, "grad_norm": 0.012336946092545986, "learning_rate": 5.0507984812754684e-05, "loss": 0.00012418095138855278, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00012, "step": 285, "tokens/total": 9284008, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 130417} +{"epoch": 1.1171875, "grad_norm": 0.4516570270061493, "learning_rate": 5.021923930849237e-05, "loss": 0.0030764073599129915, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.45, "memory/max_allocated (GiB)": 64.45, "ppl": 1.00308, "step": 286, "tokens/total": 9315984, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 130903} +{"epoch": 1.12109375, "grad_norm": 0.0067978654988110065, "learning_rate": 4.99306927511967e-05, "loss": 6.418054545065388e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.00006, "step": 287, "tokens/total": 9347376, "tokens/train_per_sec_per_gpu": 10.23, "tokens/trainable": 131343} +{"epoch": 1.125, "grad_norm": 0.5215632915496826, "learning_rate": 4.964235714846775e-05, "loss": 0.003616590052843094, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.00362, "step": 288, "tokens/total": 9377424, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 131763} +{"epoch": 1.12890625, "grad_norm": 0.011731327511370182, "learning_rate": 4.9354244499126866e-05, "loss": 6.963473424548283e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00007, "step": 289, "tokens/total": 9408104, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 132229} +{"epoch": 1.1328125, "grad_norm": 0.042008642107248306, "learning_rate": 4.90663667927174e-05, "loss": 0.00011662385077215731, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00012, "step": 290, "tokens/total": 9439944, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 132678} +{"epoch": 1.13671875, "grad_norm": 0.23219190537929535, "learning_rate": 4.877873600900581e-05, "loss": 0.000421657576225698, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00042, "step": 291, "tokens/total": 9472104, "tokens/train_per_sec_per_gpu": 6.12, "tokens/trainable": 133136} +{"epoch": 1.140625, "grad_norm": 0.004881258588284254, "learning_rate": 4.849136411748306e-05, "loss": 3.468960858299397e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00003, "step": 292, "tokens/total": 9508336, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 133608} +{"epoch": 1.14453125, "grad_norm": 0.03458632901310921, "learning_rate": 4.8204263076866574e-05, "loss": 0.0004134753253310919, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00041, "step": 293, "tokens/total": 9541552, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 134108} +{"epoch": 1.1484375, "grad_norm": 0.006113293115049601, "learning_rate": 4.791744483460251e-05, "loss": 2.4304177713929676e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00002, "step": 294, "tokens/total": 9575072, "tokens/train_per_sec_per_gpu": 6.47, "tokens/trainable": 134541} +{"epoch": 1.15234375, "grad_norm": 0.0025416675489395857, "learning_rate": 4.7630921326368736e-05, "loss": 3.34225915139541e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00003, "step": 295, "tokens/total": 9610344, "tokens/train_per_sec_per_gpu": 6.31, "tokens/trainable": 134966} +{"epoch": 1.15625, "grad_norm": 0.48059800267219543, "learning_rate": 4.7344704475577916e-05, "loss": 0.004902873653918505, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00491, "step": 296, "tokens/total": 9639728, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 135402} +{"epoch": 1.16015625, "grad_norm": 0.30979156494140625, "learning_rate": 4.705880619288153e-05, "loss": 0.004564731847494841, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.07, "memory/max_allocated (GiB)": 65.07, "ppl": 1.00458, "step": 297, "tokens/total": 9671424, "tokens/train_per_sec_per_gpu": 6.98, "tokens/trainable": 135804} +{"epoch": 1.1640625, "grad_norm": 0.04804934188723564, "learning_rate": 4.677323837567412e-05, "loss": 0.00038617278914898634, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00039, "step": 298, "tokens/total": 9700808, "tokens/train_per_sec_per_gpu": 8.82, "tokens/trainable": 136231} +{"epoch": 1.16796875, "grad_norm": 0.004330691881477833, "learning_rate": 4.6488012907598146e-05, "loss": 3.341105912113562e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00003, "step": 299, "tokens/total": 9732312, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 136705} +{"epoch": 1.171875, "grad_norm": 0.055643230676651, "learning_rate": 4.620314165804964e-05, "loss": 0.0004528402932919562, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00045, "step": 300, "tokens/total": 9764624, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 137178} +{"epoch": 1.17578125, "grad_norm": 0.02170630358159542, "learning_rate": 4.591863648168407e-05, "loss": 0.00017241938621737063, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00017, "step": 301, "tokens/total": 9798368, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 137643} +{"epoch": 1.1796875, "grad_norm": 0.25884950160980225, "learning_rate": 4.5634509217923135e-05, "loss": 0.0024294534232467413, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00243, "step": 302, "tokens/total": 9830192, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 138078} +{"epoch": 1.18359375, "grad_norm": 0.0973874107003212, "learning_rate": 4.535077169046201e-05, "loss": 0.000742073985747993, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00074, "step": 303, "tokens/total": 9863112, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 138515} +{"epoch": 1.1875, "grad_norm": 0.6080950498580933, "learning_rate": 4.506743570677743e-05, "loss": 0.00807153433561325, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.0081, "step": 304, "tokens/total": 9893192, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 138948} +{"epoch": 1.19140625, "grad_norm": 0.3515007495880127, "learning_rate": 4.478451305763618e-05, "loss": 0.007973823696374893, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00801, "step": 305, "tokens/total": 9925712, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 139408} +{"epoch": 1.1953125, "grad_norm": 0.010054406709969044, "learning_rate": 4.450201551660454e-05, "loss": 6.396780372597277e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00006, "step": 306, "tokens/total": 9955736, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 139840} +{"epoch": 1.19921875, "grad_norm": 0.1973126083612442, "learning_rate": 4.4219954839558276e-05, "loss": 0.002608177252113819, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00261, "step": 307, "tokens/total": 9987160, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 140281} +{"epoch": 1.203125, "grad_norm": 0.14571025967597961, "learning_rate": 4.393834276419352e-05, "loss": 0.0005331095890142024, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00053, "step": 308, "tokens/total": 10017360, "tokens/train_per_sec_per_gpu": 10.96, "tokens/trainable": 140776} +{"epoch": 1.20703125, "grad_norm": 0.007020301651209593, "learning_rate": 4.36571910095382e-05, "loss": 6.0661357565550134e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00006, "step": 309, "tokens/total": 10052056, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 141228} +{"epoch": 1.2109375, "grad_norm": 0.14903901517391205, "learning_rate": 4.337651127546448e-05, "loss": 0.0010226685553789139, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00102, "step": 310, "tokens/total": 10085424, "tokens/train_per_sec_per_gpu": 9.57, "tokens/trainable": 141679} +{"epoch": 1.21484375, "grad_norm": 0.16125357151031494, "learning_rate": 4.3096315242201736e-05, "loss": 0.0008049350581131876, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.18, "memory/max_allocated (GiB)": 64.18, "ppl": 1.00081, "step": 311, "tokens/total": 10116824, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 142122} +{"epoch": 1.21875, "grad_norm": 0.02003222517669201, "learning_rate": 4.2816614569850635e-05, "loss": 9.463926107855514e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00009, "step": 312, "tokens/total": 10150376, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 142561} +{"epoch": 1.22265625, "grad_norm": 0.4561345875263214, "learning_rate": 4.2537420897897864e-05, "loss": 0.0076777879148721695, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00771, "step": 313, "tokens/total": 10184432, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 143046} +{"epoch": 1.2265625, "grad_norm": 0.01813521236181259, "learning_rate": 4.225874584473174e-05, "loss": 0.00014509883476421237, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00015, "step": 314, "tokens/total": 10214848, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 143493} +{"epoch": 1.23046875, "grad_norm": 0.30256927013397217, "learning_rate": 4.19806010071587e-05, "loss": 0.0015777864027768373, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00158, "step": 315, "tokens/total": 10247952, "tokens/train_per_sec_per_gpu": 8.88, "tokens/trainable": 143956} +{"epoch": 1.234375, "grad_norm": 0.21188123524188995, "learning_rate": 4.170299795992081e-05, "loss": 0.0020995656959712505, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.0021, "step": 316, "tokens/total": 10280648, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 144411} +{"epoch": 1.23828125, "grad_norm": 0.1154693141579628, "learning_rate": 4.142594825521398e-05, "loss": 0.0009758276864886284, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00098, "step": 317, "tokens/total": 10317712, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 144892} +{"epoch": 1.2421875, "grad_norm": 0.023124031722545624, "learning_rate": 4.114946342220728e-05, "loss": 5.816884367959574e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00006, "step": 318, "tokens/total": 10351632, "tokens/train_per_sec_per_gpu": 6.85, "tokens/trainable": 145339} +{"epoch": 1.24609375, "grad_norm": 0.042994603514671326, "learning_rate": 4.087355496656321e-05, "loss": 0.0003562100464478135, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00036, "step": 319, "tokens/total": 10383336, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 145811} +{"epoch": 1.25, "grad_norm": 0.9893935918807983, "learning_rate": 4.05982343699588e-05, "loss": 0.0014869199367240071, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00149, "step": 320, "tokens/total": 10418016, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 146314} +{"epoch": 1.25390625, "grad_norm": 0.2238604724407196, "learning_rate": 4.0323513089607876e-05, "loss": 0.0017714485293254256, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00177, "step": 321, "tokens/total": 10451600, "tokens/train_per_sec_per_gpu": 7.24, "tokens/trainable": 146781} +{"epoch": 1.2578125, "grad_norm": 0.02317190170288086, "learning_rate": 4.004940255778431e-05, "loss": 0.00012166365922894329, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00012, "step": 322, "tokens/total": 10487568, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 147225} +{"epoch": 1.26171875, "grad_norm": 0.057658012956380844, "learning_rate": 3.977591418134619e-05, "loss": 0.00026608770713210106, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00027, "step": 323, "tokens/total": 10520976, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 147673} +{"epoch": 1.265625, "grad_norm": 0.003957323729991913, "learning_rate": 3.95030593412612e-05, "loss": 4.0347338654100895e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00004, "step": 324, "tokens/total": 10552776, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 148103} +{"epoch": 1.26953125, "grad_norm": 0.036238282918930054, "learning_rate": 3.923084939213296e-05, "loss": 8.569978672312573e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00009, "step": 325, "tokens/total": 10582576, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 148535} +{"epoch": 1.2734375, "grad_norm": 0.172017902135849, "learning_rate": 3.895929566172861e-05, "loss": 0.0016208906890824437, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00162, "step": 326, "tokens/total": 10615600, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 148999} +{"epoch": 1.27734375, "grad_norm": 0.12393037974834442, "learning_rate": 3.868840945050728e-05, "loss": 0.0007785677444189787, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.72, "memory/max_allocated (GiB)": 65.72, "ppl": 1.00078, "step": 327, "tokens/total": 10650688, "tokens/train_per_sec_per_gpu": 7.34, "tokens/trainable": 149431} +{"epoch": 1.28125, "grad_norm": 0.02778439410030842, "learning_rate": 3.841820203114995e-05, "loss": 0.00013942681835033, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00014, "step": 328, "tokens/total": 10681816, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 149886} +{"epoch": 1.28515625, "grad_norm": 0.0016951044090092182, "learning_rate": 3.814868464809027e-05, "loss": 1.95560569409281e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00002, "step": 329, "tokens/total": 10711728, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 150288} +{"epoch": 1.2890625, "grad_norm": 0.005374022293835878, "learning_rate": 3.787986851704667e-05, "loss": 8.254984277300537e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00008, "step": 330, "tokens/total": 10744952, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 150733} +{"epoch": 1.29296875, "grad_norm": 0.0038094925694167614, "learning_rate": 3.7611764824555654e-05, "loss": 4.6553745050914586e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00005, "step": 331, "tokens/total": 10776192, "tokens/train_per_sec_per_gpu": 9.6, "tokens/trainable": 151207} +{"epoch": 1.296875, "grad_norm": 0.001992329489439726, "learning_rate": 3.734438472750619e-05, "loss": 2.557548759796191e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00003, "step": 332, "tokens/total": 10813336, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 151694} +{"epoch": 1.30078125, "grad_norm": 0.020079022273421288, "learning_rate": 3.707773935267552e-05, "loss": 0.00011388187704142183, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.65, "memory/max_allocated (GiB)": 65.65, "ppl": 1.00011, "step": 333, "tokens/total": 10847808, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 152188} +{"epoch": 1.3046875, "grad_norm": 0.0007526230183430016, "learning_rate": 3.68118397962661e-05, "loss": 1.250477544090245e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00001, "step": 334, "tokens/total": 10879896, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 152596} +{"epoch": 1.30859375, "grad_norm": 0.0070465849712491035, "learning_rate": 3.654669712344384e-05, "loss": 1.8725721020018682e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00002, "step": 335, "tokens/total": 10908976, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 153052} +{"epoch": 1.3125, "grad_norm": 0.013822110369801521, "learning_rate": 3.628232236787763e-05, "loss": 0.00010581874084891751, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.88, "memory/max_allocated (GiB)": 64.88, "ppl": 1.00011, "step": 336, "tokens/total": 10943152, "tokens/train_per_sec_per_gpu": 8.08, "tokens/trainable": 153491} +{"epoch": 1.31640625, "grad_norm": 0.009205128066241741, "learning_rate": 3.6018726531280144e-05, "loss": 7.701353024458513e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00008, "step": 337, "tokens/total": 10978184, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 153983} +{"epoch": 1.3203125, "grad_norm": 0.06193067878484726, "learning_rate": 3.575592058295017e-05, "loss": 0.00047148013254627585, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00047, "step": 338, "tokens/total": 11010056, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 154464} +{"epoch": 1.32421875, "grad_norm": 0.0012876306427642703, "learning_rate": 3.549391545931585e-05, "loss": 1.1551320312719326e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.00001, "step": 339, "tokens/total": 11040968, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 154924} +{"epoch": 1.328125, "grad_norm": 0.036400746554136276, "learning_rate": 3.5232722063479914e-05, "loss": 0.0001456026511732489, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00015, "step": 340, "tokens/total": 11073424, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 155373} +{"epoch": 1.33203125, "grad_norm": 0.04587194696068764, "learning_rate": 3.49723512647657e-05, "loss": 0.00042199334711767733, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00042, "step": 341, "tokens/total": 11109224, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 155873} +{"epoch": 1.3359375, "grad_norm": 0.046591613441705704, "learning_rate": 3.471281389826491e-05, "loss": 6.306420982582495e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00006, "step": 342, "tokens/total": 11139856, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 156278} +{"epoch": 1.33984375, "grad_norm": 0.0029100440442562103, "learning_rate": 3.4454120764386764e-05, "loss": 1.8660541172721423e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00002, "step": 343, "tokens/total": 11175224, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 156733} +{"epoch": 1.34375, "grad_norm": 0.02106001414358616, "learning_rate": 3.4196282628408526e-05, "loss": 0.00014869184815324843, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00015, "step": 344, "tokens/total": 11207112, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 157203} +{"epoch": 1.34765625, "grad_norm": 0.07878031581640244, "learning_rate": 3.3939310220027456e-05, "loss": 0.0004994196351617575, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.0005, "step": 345, "tokens/total": 11240432, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 157707} +{"epoch": 1.3515625, "grad_norm": 0.006136393174529076, "learning_rate": 3.3683214232914404e-05, "loss": 4.266284668119624e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00004, "step": 346, "tokens/total": 11274584, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 158180} +{"epoch": 1.35546875, "grad_norm": 0.04181620851159096, "learning_rate": 3.342800532426873e-05, "loss": 0.00013535413017962128, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00014, "step": 347, "tokens/total": 11307376, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 158646} +{"epoch": 1.359375, "grad_norm": 0.09539923816919327, "learning_rate": 3.317369411437484e-05, "loss": 0.00026317001902498305, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00026, "step": 348, "tokens/total": 11340640, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 159115} +{"epoch": 1.36328125, "grad_norm": 0.0085196103900671, "learning_rate": 3.292029118616024e-05, "loss": 6.976544682402164e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00007, "step": 349, "tokens/total": 11371632, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 159538} +{"epoch": 1.3671875, "grad_norm": 0.006819752510637045, "learning_rate": 3.266780708475511e-05, "loss": 4.440713382791728e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00004, "step": 350, "tokens/total": 11404248, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 160016} +{"epoch": 1.37109375, "grad_norm": 0.0015879247803241014, "learning_rate": 3.241625231705354e-05, "loss": 6.015884991938947e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00001, "step": 351, "tokens/total": 11439552, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 160475} +{"epoch": 1.375, "grad_norm": 0.609782338142395, "learning_rate": 3.216563735127618e-05, "loss": 0.0024428521282970905, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00245, "step": 352, "tokens/total": 11470944, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 160952} +{"epoch": 1.37890625, "grad_norm": 0.008393614552915096, "learning_rate": 3.191597261653475e-05, "loss": 2.8628110158024356e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00003, "step": 353, "tokens/total": 11506304, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 161420} +{"epoch": 1.3828125, "grad_norm": 0.34703636169433594, "learning_rate": 3.166726850239794e-05, "loss": 0.0011836634948849678, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00118, "step": 354, "tokens/total": 11535424, "tokens/train_per_sec_per_gpu": 10.79, "tokens/trainable": 161868} +{"epoch": 1.38671875, "grad_norm": 0.0004933669115416706, "learning_rate": 3.141953535845912e-05, "loss": 1.1733272003766615e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00001, "step": 355, "tokens/total": 11565176, "tokens/train_per_sec_per_gpu": 10.09, "tokens/trainable": 162318} +{"epoch": 1.390625, "grad_norm": 0.001870613545179367, "learning_rate": 3.11727834939056e-05, "loss": 1.664456976868678e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.48, "memory/max_allocated (GiB)": 63.48, "ppl": 1.00002, "step": 356, "tokens/total": 11595528, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 162789} +{"epoch": 1.39453125, "grad_norm": 0.29448384046554565, "learning_rate": 3.092702317708967e-05, "loss": 0.005139114800840616, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00515, "step": 357, "tokens/total": 11627248, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 163254} +{"epoch": 1.3984375, "grad_norm": 0.0009000370046123862, "learning_rate": 3.0682264635101276e-05, "loss": 1.2799983778677415e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00001, "step": 358, "tokens/total": 11659440, "tokens/train_per_sec_per_gpu": 8.75, "tokens/trainable": 163715} +{"epoch": 1.40234375, "grad_norm": 0.01973443105816841, "learning_rate": 3.0438518053342407e-05, "loss": 0.00010823743650689721, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00011, "step": 359, "tokens/total": 11693288, "tokens/train_per_sec_per_gpu": 7.69, "tokens/trainable": 164197} +{"epoch": 1.40625, "grad_norm": 0.0019913637079298496, "learning_rate": 3.0195793575103266e-05, "loss": 2.0101848349440843e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00002, "step": 360, "tokens/total": 11730888, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 164661} +{"epoch": 1.41015625, "grad_norm": 0.2250841110944748, "learning_rate": 2.9954101301140146e-05, "loss": 0.0007520507206209004, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00075, "step": 361, "tokens/total": 11765656, "tokens/train_per_sec_per_gpu": 6.79, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.005048481747508049, "learning_rate": 2.9713451289255123e-05, "loss": 5.007957952329889e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00005, "step": 362, "tokens/total": 11793456, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 165552} +{"epoch": 1.41796875, "grad_norm": 0.03970780223608017, "learning_rate": 2.9473853553877484e-05, "loss": 8.779361814958975e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00009, "step": 363, "tokens/total": 11826280, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 166026} +{"epoch": 1.421875, "grad_norm": 0.00795234739780426, "learning_rate": 2.9235318065647e-05, "loss": 7.876128802308813e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00008, "step": 364, "tokens/total": 11863552, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 166486} +{"epoch": 1.42578125, "grad_norm": 0.0003213443560525775, "learning_rate": 2.8997854750998964e-05, "loss": 5.136795152793638e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00001, "step": 365, "tokens/total": 11898616, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 166959} +{"epoch": 1.4296875, "grad_norm": 0.02503248117864132, "learning_rate": 2.8761473491751258e-05, "loss": 0.00016012514242902398, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.86, "memory/max_allocated (GiB)": 63.86, "ppl": 1.00016, "step": 366, "tokens/total": 11930728, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 167394} +{"epoch": 1.43359375, "grad_norm": 0.0068013849668204784, "learning_rate": 2.8526184124692883e-05, "loss": 1.8873315639211796e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00002, "step": 367, "tokens/total": 11964976, "tokens/train_per_sec_per_gpu": 6.3, "tokens/trainable": 167832} +{"epoch": 1.4375, "grad_norm": 0.06828402727842331, "learning_rate": 2.829199644117484e-05, "loss": 0.0002046966110356152, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.0002, "step": 368, "tokens/total": 11996696, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 168242} +{"epoch": 1.44140625, "grad_norm": 0.003885376499965787, "learning_rate": 2.8058920186702553e-05, "loss": 2.2841979443910532e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.00002, "step": 369, "tokens/total": 12033432, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 168715} +{"epoch": 1.4453125, "grad_norm": 0.00030706211691722274, "learning_rate": 2.782696506053033e-05, "loss": 6.519357611978194e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00001, "step": 370, "tokens/total": 12067616, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 169147} +{"epoch": 1.44921875, "grad_norm": 0.3798683285713196, "learning_rate": 2.7596140715257824e-05, "loss": 0.005445912946015596, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00546, "step": 371, "tokens/total": 12099008, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 169628} +{"epoch": 1.453125, "grad_norm": 0.007158064749091864, "learning_rate": 2.7366456756428184e-05, "loss": 5.488914030138403e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00005, "step": 372, "tokens/total": 12131304, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 170085} +{"epoch": 1.45703125, "grad_norm": 0.014884118922054768, "learning_rate": 2.7137922742128486e-05, "loss": 9.17862489586696e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00009, "step": 373, "tokens/total": 12164104, "tokens/train_per_sec_per_gpu": 9.02, "tokens/trainable": 170584} +{"epoch": 1.4609375, "grad_norm": 1.551958441734314, "learning_rate": 2.691054818259188e-05, "loss": 0.012517700903117657, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.0126, "step": 374, "tokens/total": 12197776, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 171058} +{"epoch": 1.46484375, "grad_norm": 0.13201439380645752, "learning_rate": 2.6684342539801933e-05, "loss": 0.0004634306824300438, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.37, "memory/max_allocated (GiB)": 63.37, "ppl": 1.00046, "step": 375, "tokens/total": 12228064, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 171518} +{"epoch": 1.46875, "grad_norm": 0.008519172668457031, "learning_rate": 2.645931522709877e-05, "loss": 3.326379737700336e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00003, "step": 376, "tokens/total": 12261688, "tokens/train_per_sec_per_gpu": 10.56, "tokens/trainable": 172007} +{"epoch": 1.47265625, "grad_norm": 0.05487489327788353, "learning_rate": 2.6235475608787365e-05, "loss": 0.00030856518424116075, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00031, "step": 377, "tokens/total": 12294240, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 172468} +{"epoch": 1.4765625, "grad_norm": 0.24260196089744568, "learning_rate": 2.6012832999747916e-05, "loss": 0.0009716198546811938, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00097, "step": 378, "tokens/total": 12324528, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 172923} +{"epoch": 1.48046875, "grad_norm": 1.0233582258224487, "learning_rate": 2.579139666504821e-05, "loss": 0.022961651906371117, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.02323, "step": 379, "tokens/total": 12360288, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 173370} +{"epoch": 1.484375, "grad_norm": 0.017727080732584, "learning_rate": 2.557117581955798e-05, "loss": 8.837903442326933e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00009, "step": 380, "tokens/total": 12391216, "tokens/train_per_sec_per_gpu": 8.93, "tokens/trainable": 173837} +{"epoch": 1.48828125, "grad_norm": 1.5245037078857422, "learning_rate": 2.5352179627565532e-05, "loss": 0.002032787073403597, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00203, "step": 381, "tokens/total": 12423624, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 174294} +{"epoch": 1.4921875, "grad_norm": 0.0008745007216930389, "learning_rate": 2.5134417202396277e-05, "loss": 1.2568780221045017e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00001, "step": 382, "tokens/total": 12456032, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 174701} +{"epoch": 1.49609375, "grad_norm": 0.2989595830440521, "learning_rate": 2.491789760603361e-05, "loss": 0.0018972842954099178, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.0019, "step": 383, "tokens/total": 12489336, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 175131} +{"epoch": 1.5, "grad_norm": 0.010640951804816723, "learning_rate": 2.4702629848741764e-05, "loss": 5.099148620502092e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00005, "step": 384, "tokens/total": 12521944, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 175586} +{"epoch": 1.50390625, "grad_norm": 0.008532666601240635, "learning_rate": 2.4488622888690785e-05, "loss": 6.419347482733428e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00006, "step": 385, "tokens/total": 12554024, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 176026} +{"epoch": 1.5078125, "grad_norm": 0.006018294487148523, "learning_rate": 2.427588563158384e-05, "loss": 3.59542973455973e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00004, "step": 386, "tokens/total": 12587104, "tokens/train_per_sec_per_gpu": 7.06, "tokens/trainable": 176512} +{"epoch": 1.51171875, "grad_norm": 0.05730123817920685, "learning_rate": 2.406442693028651e-05, "loss": 0.0002858355001080781, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.4, "memory/max_allocated (GiB)": 65.4, "ppl": 1.00029, "step": 387, "tokens/total": 12617456, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 176943} +{"epoch": 1.515625, "grad_norm": 0.007924120873212814, "learning_rate": 2.3854255584458547e-05, "loss": 4.257548425812274e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00004, "step": 388, "tokens/total": 12652696, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 177370} +{"epoch": 1.51953125, "grad_norm": 0.006449631415307522, "learning_rate": 2.3645380340187508e-05, "loss": 5.727010648115538e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.00006, "step": 389, "tokens/total": 12685792, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 177851} +{"epoch": 1.5234375, "grad_norm": 0.00222744676284492, "learning_rate": 2.3437809889624914e-05, "loss": 2.7026579118683003e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00003, "step": 390, "tokens/total": 12716120, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 178310} +{"epoch": 1.52734375, "grad_norm": 0.003044383367523551, "learning_rate": 2.3231552870624487e-05, "loss": 5.3861735068494454e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00005, "step": 391, "tokens/total": 12746216, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 178736} +{"epoch": 1.53125, "grad_norm": 0.0798015221953392, "learning_rate": 2.3026617866382657e-05, "loss": 0.0005172250675968826, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00052, "step": 392, "tokens/total": 12778592, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 179233} +{"epoch": 1.53515625, "grad_norm": 0.016317537054419518, "learning_rate": 2.2823013405081507e-05, "loss": 0.0001181542465928942, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00012, "step": 393, "tokens/total": 12812008, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 179730} +{"epoch": 1.5390625, "grad_norm": 0.0018328968435525894, "learning_rate": 2.2620747959533722e-05, "loss": 2.382201091677416e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00002, "step": 394, "tokens/total": 12844264, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 180227} +{"epoch": 1.54296875, "grad_norm": 0.017471909523010254, "learning_rate": 2.2419829946830123e-05, "loss": 9.34685449465178e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00009, "step": 395, "tokens/total": 12878328, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 180687} +{"epoch": 1.546875, "grad_norm": 0.05822361260652542, "learning_rate": 2.2220267727989325e-05, "loss": 0.00067543750628829, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.94, "memory/max_allocated (GiB)": 64.94, "ppl": 1.00068, "step": 396, "tokens/total": 12907912, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 181141} +{"epoch": 1.55078125, "grad_norm": 0.5224599838256836, "learning_rate": 2.202206960760984e-05, "loss": 0.006447078660130501, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00647, "step": 397, "tokens/total": 12941968, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 181648} +{"epoch": 1.5546875, "grad_norm": 0.6872231364250183, "learning_rate": 2.182524383352446e-05, "loss": 0.0005171783268451691, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00052, "step": 398, "tokens/total": 12975240, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 182109} +{"epoch": 1.55859375, "grad_norm": 0.04579411447048187, "learning_rate": 2.1629798596457056e-05, "loss": 0.0005271499394439161, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.88, "memory/max_allocated (GiB)": 63.88, "ppl": 1.00053, "step": 399, "tokens/total": 13006912, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 182559} +{"epoch": 1.5625, "grad_norm": 0.6060359477996826, "learning_rate": 2.1435742029681725e-05, "loss": 0.002453774679452181, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00246, "step": 400, "tokens/total": 13037640, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 182985} +{"epoch": 1.56640625, "grad_norm": 0.006364879198372364, "learning_rate": 2.124308220868431e-05, "loss": 7.379594899248332e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.43, "memory/max_allocated (GiB)": 65.43, "ppl": 1.00007, "step": 401, "tokens/total": 13070608, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 183440} +{"epoch": 1.5703125, "grad_norm": 0.010428167879581451, "learning_rate": 2.105182715082638e-05, "loss": 7.932306471047923e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.00008, "step": 402, "tokens/total": 13101784, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 183917} +{"epoch": 1.57421875, "grad_norm": 0.020056406036019325, "learning_rate": 2.0861984815011552e-05, "loss": 0.00018365880532655865, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00018, "step": 403, "tokens/total": 13133528, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 184348} +{"epoch": 1.578125, "grad_norm": 0.015494188293814659, "learning_rate": 2.0673563101354323e-05, "loss": 0.00017971564375329763, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00018, "step": 404, "tokens/total": 13167088, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 184786} +{"epoch": 1.58203125, "grad_norm": 0.014627004973590374, "learning_rate": 2.0486569850851317e-05, "loss": 9.925432095769793e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.0001, "step": 405, "tokens/total": 13200392, "tokens/train_per_sec_per_gpu": 9.58, "tokens/trainable": 185249} +{"epoch": 1.5859375, "grad_norm": 0.041086502373218536, "learning_rate": 2.0301012845054956e-05, "loss": 0.00047306064516305923, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.71, "memory/max_allocated (GiB)": 65.71, "ppl": 1.00047, "step": 406, "tokens/total": 13235488, "tokens/train_per_sec_per_gpu": 6.76, "tokens/trainable": 185680} +{"epoch": 1.58984375, "grad_norm": 0.04578345641493797, "learning_rate": 2.011689980574966e-05, "loss": 0.00039224480860866606, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00039, "step": 407, "tokens/total": 13269416, "tokens/train_per_sec_per_gpu": 7.65, "tokens/trainable": 186100} +{"epoch": 1.59375, "grad_norm": 0.2694893479347229, "learning_rate": 1.993423839463052e-05, "loss": 0.002254810417070985, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.98, "memory/max_allocated (GiB)": 64.98, "ppl": 1.00226, "step": 408, "tokens/total": 13301360, "tokens/train_per_sec_per_gpu": 10.25, "tokens/trainable": 186565} +{"epoch": 1.59765625, "grad_norm": 0.013135059736669064, "learning_rate": 1.975303621298445e-05, "loss": 9.835843229666352e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.87, "memory/max_allocated (GiB)": 65.87, "ppl": 1.0001, "step": 409, "tokens/total": 13336392, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 187017} +{"epoch": 1.6015625, "grad_norm": 0.009020629338920116, "learning_rate": 1.957330080137385e-05, "loss": 8.175714174285531e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.59, "memory/max_allocated (GiB)": 63.59, "ppl": 1.00008, "step": 410, "tokens/total": 13366072, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 187468} +{"epoch": 1.60546875, "grad_norm": 0.004432655405253172, "learning_rate": 1.9395039639322864e-05, "loss": 7.066434773150831e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00007, "step": 411, "tokens/total": 13400832, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 187961} +{"epoch": 1.609375, "grad_norm": 0.0015715461922809482, "learning_rate": 1.9218260145006073e-05, "loss": 1.977753890969325e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00002, "step": 412, "tokens/total": 13434920, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 188447} +{"epoch": 1.61328125, "grad_norm": 0.038416385650634766, "learning_rate": 1.904296967493982e-05, "loss": 0.00019375480769667774, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.06, "memory/max_allocated (GiB)": 64.06, "ppl": 1.00019, "step": 413, "tokens/total": 13466512, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 188903} +{"epoch": 1.6171875, "grad_norm": 0.20811685919761658, "learning_rate": 1.8869175523676064e-05, "loss": 0.001197037287056446, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.0012, "step": 414, "tokens/total": 13500352, "tokens/train_per_sec_per_gpu": 8.77, "tokens/trainable": 189398} +{"epoch": 1.62109375, "grad_norm": 0.3787010610103607, "learning_rate": 1.869688492349885e-05, "loss": 0.003367267781868577, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00337, "step": 415, "tokens/total": 13532448, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 189850} +{"epoch": 1.625, "grad_norm": 0.10000713914632797, "learning_rate": 1.85261050441233e-05, "loss": 0.0007395246648229659, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00074, "step": 416, "tokens/total": 13565648, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 190306} +{"epoch": 1.62890625, "grad_norm": 0.17827902734279633, "learning_rate": 1.8356842992397304e-05, "loss": 0.0024386453442275524, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00244, "step": 417, "tokens/total": 13597600, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 190746} +{"epoch": 1.6328125, "grad_norm": 0.023929085582494736, "learning_rate": 1.8189105812005714e-05, "loss": 0.00016644086281303316, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.06, "memory/max_allocated (GiB)": 66.06, "ppl": 1.00017, "step": 418, "tokens/total": 13630456, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 191156} +{"epoch": 1.63671875, "grad_norm": 0.24529483914375305, "learning_rate": 1.802290048317732e-05, "loss": 0.001444352208636701, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00145, "step": 419, "tokens/total": 13660896, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 191573} +{"epoch": 1.640625, "grad_norm": 0.024910353124141693, "learning_rate": 1.785823392239424e-05, "loss": 0.00015417387476190925, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00015, "step": 420, "tokens/total": 13693176, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 192014} +{"epoch": 1.64453125, "grad_norm": 0.008049636147916317, "learning_rate": 1.7695112982104225e-05, "loss": 9.353259520139545e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00009, "step": 421, "tokens/total": 13725400, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 192453} +{"epoch": 1.6484375, "grad_norm": 0.0018950393423438072, "learning_rate": 1.7533544450435433e-05, "loss": 3.428809577599168e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.19, "memory/max_allocated (GiB)": 63.19, "ppl": 1.00003, "step": 422, "tokens/total": 13752752, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 192886} +{"epoch": 1.65234375, "grad_norm": 0.004491644911468029, "learning_rate": 1.7373535050913946e-05, "loss": 2.4869688786566257e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00002, "step": 423, "tokens/total": 13786136, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 193323} +{"epoch": 1.65625, "grad_norm": 0.020174557343125343, "learning_rate": 1.721509144218405e-05, "loss": 0.00017360656056553125, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.23, "memory/max_allocated (GiB)": 63.23, "ppl": 1.00017, "step": 424, "tokens/total": 13815904, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 193753} +{"epoch": 1.66015625, "grad_norm": 0.002659064019098878, "learning_rate": 1.705822021773101e-05, "loss": 2.5093699150602333e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.41, "memory/max_allocated (GiB)": 65.41, "ppl": 1.00003, "step": 425, "tokens/total": 13847976, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 194240} +{"epoch": 1.6640625, "grad_norm": 0.015247771516442299, "learning_rate": 1.69029279056068e-05, "loss": 0.00011613031529122964, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.00012, "step": 426, "tokens/total": 13883200, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 194680} +{"epoch": 1.66796875, "grad_norm": 0.008562522940337658, "learning_rate": 1.6749220968158415e-05, "loss": 3.078163354075514e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00003, "step": 427, "tokens/total": 13916080, "tokens/train_per_sec_per_gpu": 10.19, "tokens/trainable": 195186} +{"epoch": 1.671875, "grad_norm": 0.019507061690092087, "learning_rate": 1.659710580175893e-05, "loss": 0.00012566123041324317, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00013, "step": 428, "tokens/total": 13946448, "tokens/train_per_sec_per_gpu": 9.95, "tokens/trainable": 195693} +{"epoch": 1.67578125, "grad_norm": 0.0286449883133173, "learning_rate": 1.644658873654133e-05, "loss": 0.0001270112843485549, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.00013, "step": 429, "tokens/total": 13975696, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 196130} +{"epoch": 1.6796875, "grad_norm": 0.0027436986565589905, "learning_rate": 1.629767603613508e-05, "loss": 2.5540519345668145e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00003, "step": 430, "tokens/total": 14002616, "tokens/train_per_sec_per_gpu": 7.46, "tokens/trainable": 196530} +{"epoch": 1.68359375, "grad_norm": 0.005728337448090315, "learning_rate": 1.615037389740547e-05, "loss": 7.666947203688323e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00008, "step": 431, "tokens/total": 14034848, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 196974} +{"epoch": 1.6875, "grad_norm": 0.015160380862653255, "learning_rate": 1.600468845019576e-05, "loss": 0.0001413428835803643, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.95, "memory/max_allocated (GiB)": 65.95, "ppl": 1.00014, "step": 432, "tokens/total": 14069032, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 197454} +{"epoch": 1.69140625, "grad_norm": 0.0016189177986234426, "learning_rate": 1.5860625757072092e-05, "loss": 1.6598463844275102e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00002, "step": 433, "tokens/total": 14101296, "tokens/train_per_sec_per_gpu": 7.31, "tokens/trainable": 197902} +{"epoch": 1.6953125, "grad_norm": 0.016576368361711502, "learning_rate": 1.571819181307116e-05, "loss": 0.00014288499369286, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00014, "step": 434, "tokens/total": 14135880, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 198374} +{"epoch": 1.69921875, "grad_norm": 0.007080455310642719, "learning_rate": 1.557739254545075e-05, "loss": 0.00011772182915592566, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00012, "step": 435, "tokens/total": 14166040, "tokens/train_per_sec_per_gpu": 10.46, "tokens/trainable": 198846} +{"epoch": 1.703125, "grad_norm": 0.14155490696430206, "learning_rate": 1.543823381344311e-05, "loss": 0.0004022984066978097, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.0004, "step": 436, "tokens/total": 14196032, "tokens/train_per_sec_per_gpu": 10.56, "tokens/trainable": 199320} +{"epoch": 1.70703125, "grad_norm": 0.09422221779823303, "learning_rate": 1.5300721408011114e-05, "loss": 0.0002281241468153894, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.75, "memory/max_allocated (GiB)": 65.75, "ppl": 1.00023, "step": 437, "tokens/total": 14228008, "tokens/train_per_sec_per_gpu": 10.06, "tokens/trainable": 199777} +{"epoch": 1.7109375, "grad_norm": 0.05622388422489166, "learning_rate": 1.5164861051607254e-05, "loss": 0.000150593594298698, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.76, "memory/max_allocated (GiB)": 65.76, "ppl": 1.00015, "step": 438, "tokens/total": 14261744, "tokens/train_per_sec_per_gpu": 7.24, "tokens/trainable": 200194} +{"epoch": 1.71484375, "grad_norm": 0.003388284705579281, "learning_rate": 1.5030658397935521e-05, "loss": 4.470767089514993e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00004, "step": 439, "tokens/total": 14295208, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 200663} +{"epoch": 1.71875, "grad_norm": 0.00837083999067545, "learning_rate": 1.4898119031716104e-05, "loss": 2.7164716811967082e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00003, "step": 440, "tokens/total": 14327264, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 201121} +{"epoch": 1.72265625, "grad_norm": 0.46438151597976685, "learning_rate": 1.476724846845306e-05, "loss": 0.0018934133695438504, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.0019, "step": 441, "tokens/total": 14363088, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 201598} +{"epoch": 1.7265625, "grad_norm": 0.15076619386672974, "learning_rate": 1.463805215420471e-05, "loss": 0.001066836528480053, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00107, "step": 442, "tokens/total": 14396088, "tokens/train_per_sec_per_gpu": 8.39, "tokens/trainable": 202068} +{"epoch": 1.73046875, "grad_norm": 0.27290335297584534, "learning_rate": 1.451053546535705e-05, "loss": 0.0008530269842594862, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.27, "memory/max_allocated (GiB)": 65.27, "ppl": 1.00085, "step": 443, "tokens/total": 14429232, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 202524} +{"epoch": 1.734375, "grad_norm": 0.0008735352894291282, "learning_rate": 1.438470370840001e-05, "loss": 1.5249725038302131e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.93, "memory/max_allocated (GiB)": 64.93, "ppl": 1.00002, "step": 444, "tokens/total": 14463264, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 202996} +{"epoch": 1.73828125, "grad_norm": 0.06571941077709198, "learning_rate": 1.4260562119706606e-05, "loss": 0.0003507383225951344, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00035, "step": 445, "tokens/total": 14493552, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 203470} +{"epoch": 1.7421875, "grad_norm": 0.006709395907819271, "learning_rate": 1.413811586531508e-05, "loss": 5.889421663596295e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00006, "step": 446, "tokens/total": 14525240, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 203919} +{"epoch": 1.74609375, "grad_norm": 0.06619402021169662, "learning_rate": 1.4017370040713884e-05, "loss": 0.00026090024039149284, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00026, "step": 447, "tokens/total": 14559560, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 204376} +{"epoch": 1.75, "grad_norm": 0.002651191083714366, "learning_rate": 1.3898329670629645e-05, "loss": 3.0386925573111512e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00003, "step": 448, "tokens/total": 14594160, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 204821} +{"epoch": 1.75390625, "grad_norm": 0.00048632369725964963, "learning_rate": 1.3780999708818058e-05, "loss": 8.518230970366858e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00001, "step": 449, "tokens/total": 14624224, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 205271} +{"epoch": 1.7578125, "grad_norm": 0.0009977573063224554, "learning_rate": 1.3665385037857758e-05, "loss": 1.3409569874056615e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00001, "step": 450, "tokens/total": 14654984, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 205723} +{"epoch": 1.76171875, "grad_norm": 0.004043437074869871, "learning_rate": 1.3551490468947126e-05, "loss": 3.6525838368106633e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.61, "memory/max_allocated (GiB)": 63.61, "ppl": 1.00004, "step": 451, "tokens/total": 14685440, "tokens/train_per_sec_per_gpu": 9.46, "tokens/trainable": 206169} +{"epoch": 1.765625, "grad_norm": 0.38881200551986694, "learning_rate": 1.3439320741704075e-05, "loss": 0.0007556588971056044, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00076, "step": 452, "tokens/total": 14716120, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 206592} +{"epoch": 1.76953125, "grad_norm": 0.13541831076145172, "learning_rate": 1.3328880523968808e-05, "loss": 0.00023911698372103274, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00024, "step": 453, "tokens/total": 14748424, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 207055} +{"epoch": 1.7734375, "grad_norm": 0.06784176826477051, "learning_rate": 1.3220174411609587e-05, "loss": 4.482249278225936e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00004, "step": 454, "tokens/total": 14781256, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 207494} +{"epoch": 1.77734375, "grad_norm": 0.005231303162872791, "learning_rate": 1.3113206928331471e-05, "loss": 4.324327892391011e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 62.91, "memory/max_allocated (GiB)": 62.91, "ppl": 1.00004, "step": 455, "tokens/total": 14809552, "tokens/train_per_sec_per_gpu": 9.72, "tokens/trainable": 207956} +{"epoch": 1.78125, "grad_norm": 0.001496646786108613, "learning_rate": 1.300798252548806e-05, "loss": 1.4438846847042441e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00001, "step": 456, "tokens/total": 14841808, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 208408} +{"epoch": 1.78515625, "grad_norm": 0.003030630527064204, "learning_rate": 1.2904505581896265e-05, "loss": 2.5974426534958184e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00003, "step": 457, "tokens/total": 14875896, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 208889} +{"epoch": 1.7890625, "grad_norm": 0.006538459565490484, "learning_rate": 1.2802780403654082e-05, "loss": 8.120203710859641e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00008, "step": 458, "tokens/total": 14908424, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 209336} +{"epoch": 1.79296875, "grad_norm": 0.08550907671451569, "learning_rate": 1.2702811223961408e-05, "loss": 0.0006344741559587419, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.00063, "step": 459, "tokens/total": 14939168, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 209797} +{"epoch": 1.796875, "grad_norm": 0.0011768069816753268, "learning_rate": 1.2604602202943861e-05, "loss": 1.5765819625812583e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00002, "step": 460, "tokens/total": 14969872, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 210240} +{"epoch": 1.80078125, "grad_norm": 0.008283274248242378, "learning_rate": 1.2508157427479686e-05, "loss": 7.841418846510351e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00008, "step": 461, "tokens/total": 15001472, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 210702} +{"epoch": 1.8046875, "grad_norm": 0.05654269456863403, "learning_rate": 1.2413480911029655e-05, "loss": 0.00012632086873054504, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00013, "step": 462, "tokens/total": 15030592, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 211140} +{"epoch": 1.80859375, "grad_norm": 0.0018088241340592504, "learning_rate": 1.2320576593470082e-05, "loss": 2.322709588042926e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.00002, "step": 463, "tokens/total": 15063648, "tokens/train_per_sec_per_gpu": 8.91, "tokens/trainable": 211575} +{"epoch": 1.8125, "grad_norm": 0.29554420709609985, "learning_rate": 1.2229448340928828e-05, "loss": 0.0020614617969840765, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00206, "step": 464, "tokens/total": 15098248, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 212083} +{"epoch": 1.81640625, "grad_norm": 0.0006444689352065325, "learning_rate": 1.2140099945624458e-05, "loss": 9.55530867940979e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00001, "step": 465, "tokens/total": 15133216, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 212558} +{"epoch": 1.8203125, "grad_norm": 0.007620002143085003, "learning_rate": 1.205253512570841e-05, "loss": 4.336850906838663e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00004, "step": 466, "tokens/total": 15165416, "tokens/train_per_sec_per_gpu": 9.23, "tokens/trainable": 213014} +{"epoch": 1.82421875, "grad_norm": 0.031739942729473114, "learning_rate": 1.1966757525110255e-05, "loss": 0.00011839171929750592, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.38, "memory/max_allocated (GiB)": 63.38, "ppl": 1.00012, "step": 467, "tokens/total": 15193688, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 213465} +{"epoch": 1.828125, "grad_norm": 0.008899337612092495, "learning_rate": 1.1882770713386095e-05, "loss": 4.8849975428311154e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00005, "step": 468, "tokens/total": 15228600, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 213940} +{"epoch": 1.83203125, "grad_norm": 0.013055306859314442, "learning_rate": 1.180057818556998e-05, "loss": 8.131675713229924e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.00008, "step": 469, "tokens/total": 15262192, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 214415} +{"epoch": 1.8359375, "grad_norm": 0.14848650991916656, "learning_rate": 1.1720183362028494e-05, "loss": 0.0002001065731747076, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.0002, "step": 470, "tokens/total": 15294920, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 214880} +{"epoch": 1.83984375, "grad_norm": 0.8877183198928833, "learning_rate": 1.1641589588318387e-05, "loss": 0.0012834252556785941, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00128, "step": 471, "tokens/total": 15327432, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 215334} +{"epoch": 1.84375, "grad_norm": 0.0013438811292871833, "learning_rate": 1.1564800135047418e-05, "loss": 1.223660001414828e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00001, "step": 472, "tokens/total": 15359928, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 215822} +{"epoch": 1.84765625, "grad_norm": 0.004304856061935425, "learning_rate": 1.148981819773816e-05, "loss": 4.274465754861012e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00004, "step": 473, "tokens/total": 15391400, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 216239} +{"epoch": 1.8515625, "grad_norm": 0.00185546581633389, "learning_rate": 1.1416646896695086e-05, "loss": 1.3910274901718367e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00001, "step": 474, "tokens/total": 15424144, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 216691} +{"epoch": 1.85546875, "grad_norm": 0.0015478517161682248, "learning_rate": 1.1345289276874717e-05, "loss": 1.8998085579369217e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00002, "step": 475, "tokens/total": 15457680, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 217162} +{"epoch": 1.859375, "grad_norm": 0.03329206630587578, "learning_rate": 1.1275748307758873e-05, "loss": 0.0001714053942123428, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.00017, "step": 476, "tokens/total": 15492056, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 217638} +{"epoch": 1.86328125, "grad_norm": 0.0043160622008144855, "learning_rate": 1.1208026883231147e-05, "loss": 3.4052878618240356e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.84, "memory/max_allocated (GiB)": 63.84, "ppl": 1.00003, "step": 477, "tokens/total": 15521544, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 218053} +{"epoch": 1.8671875, "grad_norm": 0.004775839392095804, "learning_rate": 1.1142127821456433e-05, "loss": 3.545919025782496e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.00004, "step": 478, "tokens/total": 15553320, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 218503} +{"epoch": 1.87109375, "grad_norm": 0.031448688358068466, "learning_rate": 1.1078053864763674e-05, "loss": 0.0001421907072653994, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00014, "step": 479, "tokens/total": 15586208, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 218964} +{"epoch": 1.875, "grad_norm": 0.002149938838556409, "learning_rate": 1.1015807679531756e-05, "loss": 2.7630394470179453e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00003, "step": 480, "tokens/total": 15616432, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 219412} +{"epoch": 1.87890625, "grad_norm": 0.01128440536558628, "learning_rate": 1.0955391856078528e-05, "loss": 5.836049240315333e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00006, "step": 481, "tokens/total": 15650056, "tokens/train_per_sec_per_gpu": 10.13, "tokens/trainable": 219886} +{"epoch": 1.8828125, "grad_norm": 0.31818264722824097, "learning_rate": 1.0896808908553007e-05, "loss": 0.0021164221689105034, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00212, "step": 482, "tokens/total": 15680336, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 220343} +{"epoch": 1.88671875, "grad_norm": 0.0016441374318674207, "learning_rate": 1.0840061274830763e-05, "loss": 2.6134814106626436e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.00003, "step": 483, "tokens/total": 15711552, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 220825} +{"epoch": 1.890625, "grad_norm": 0.02619369700551033, "learning_rate": 1.0785151316412473e-05, "loss": 2.566206967458129e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.02, "memory/max_allocated (GiB)": 65.02, "ppl": 1.00003, "step": 484, "tokens/total": 15744664, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 221282} +{"epoch": 1.89453125, "grad_norm": 0.057852599769830704, "learning_rate": 1.0732081318325639e-05, "loss": 5.3800289606442675e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00005, "step": 485, "tokens/total": 15782800, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 221780} +{"epoch": 1.8984375, "grad_norm": 0.01138694491237402, "learning_rate": 1.0680853489029501e-05, "loss": 5.6348537327721715e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.33, "memory/max_allocated (GiB)": 65.33, "ppl": 1.00006, "step": 486, "tokens/total": 15817368, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 222210} +{"epoch": 1.90234375, "grad_norm": 0.01869559846818447, "learning_rate": 1.0631469960323152e-05, "loss": 8.574831736041233e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00009, "step": 487, "tokens/total": 15850408, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 222646} +{"epoch": 1.90625, "grad_norm": 0.02715475670993328, "learning_rate": 1.0583932787256783e-05, "loss": 0.00018742226529866457, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.00019, "step": 488, "tokens/total": 15881448, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 223066} +{"epoch": 1.91015625, "grad_norm": 0.0014091372722759843, "learning_rate": 1.0538243948046206e-05, "loss": 1.511994287284324e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00002, "step": 489, "tokens/total": 15913736, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 223512} +{"epoch": 1.9140625, "grad_norm": 0.00047993820044212043, "learning_rate": 1.0494405343990523e-05, "loss": 7.870851732150186e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00001, "step": 490, "tokens/total": 15944568, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 223956} +{"epoch": 1.91796875, "grad_norm": 0.002436699578538537, "learning_rate": 1.0452418799392985e-05, "loss": 1.7172254956676625e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00002, "step": 491, "tokens/total": 15976352, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 224416} +{"epoch": 1.921875, "grad_norm": 0.04167763888835907, "learning_rate": 1.0412286061485102e-05, "loss": 9.385198791278526e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00009, "step": 492, "tokens/total": 16010392, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 224883} +{"epoch": 1.92578125, "grad_norm": 0.00492989644408226, "learning_rate": 1.03740088003539e-05, "loss": 2.8051807021256536e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.35, "memory/max_allocated (GiB)": 63.35, "ppl": 1.00003, "step": 493, "tokens/total": 16039176, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 225324} +{"epoch": 1.9296875, "grad_norm": 0.005514275748282671, "learning_rate": 1.0337588608872463e-05, "loss": 3.225058753741905e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.00003, "step": 494, "tokens/total": 16067928, "tokens/train_per_sec_per_gpu": 10.34, "tokens/trainable": 225783} +{"epoch": 1.93359375, "grad_norm": 0.004746213089674711, "learning_rate": 1.0303027002633622e-05, "loss": 2.847041832865216e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00003, "step": 495, "tokens/total": 16100264, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 226281} +{"epoch": 1.9375, "grad_norm": 0.19102972745895386, "learning_rate": 1.0270325419886884e-05, "loss": 0.0010439532343298197, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00104, "step": 496, "tokens/total": 16132504, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 226735} +{"epoch": 1.94140625, "grad_norm": 0.015285369008779526, "learning_rate": 1.0239485221478599e-05, "loss": 7.363063923548907e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00007, "step": 497, "tokens/total": 16167064, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 227198} +{"epoch": 1.9453125, "grad_norm": 0.0005617289571091533, "learning_rate": 1.0210507690795292e-05, "loss": 8.430692105321214e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00001, "step": 498, "tokens/total": 16198584, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 227623} +{"epoch": 1.94921875, "grad_norm": 0.015753502026200294, "learning_rate": 1.0183394033710305e-05, "loss": 6.033886893419549e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00006, "step": 499, "tokens/total": 16229584, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 228074} +{"epoch": 1.953125, "grad_norm": 0.01765376701951027, "learning_rate": 1.0158145378533583e-05, "loss": 0.00011063885904150084, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 63.78, "memory/max_allocated (GiB)": 63.78, "ppl": 1.00011, "step": 500, "tokens/total": 16260232, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 228541} +{"epoch": 1.95703125, "grad_norm": 0.0029612171929329634, "learning_rate": 1.0134762775964726e-05, "loss": 1.258346310351044e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00001, "step": 501, "tokens/total": 16292352, "tokens/train_per_sec_per_gpu": 9.71, "tokens/trainable": 229031} +{"epoch": 1.9609375, "grad_norm": 0.011309136636555195, "learning_rate": 1.0113247199049278e-05, "loss": 4.731218359665945e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00005, "step": 502, "tokens/total": 16325424, "tokens/train_per_sec_per_gpu": 9.53, "tokens/trainable": 229512} +{"epoch": 1.96484375, "grad_norm": 0.015265963040292263, "learning_rate": 1.0093599543138205e-05, "loss": 9.710936865303665e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.0001, "step": 503, "tokens/total": 16359184, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 229989} +{"epoch": 1.96875, "grad_norm": 0.01699521765112877, "learning_rate": 1.0075820625850675e-05, "loss": 9.964763739844784e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.0001, "step": 504, "tokens/total": 16395416, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 230447} +{"epoch": 1.97265625, "grad_norm": 0.02490832656621933, "learning_rate": 1.0059911187040013e-05, "loss": 6.202970689628273e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00006, "step": 505, "tokens/total": 16428088, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 230879} +{"epoch": 1.9765625, "grad_norm": 0.0005764731322415173, "learning_rate": 1.0045871888762893e-05, "loss": 7.863945938879624e-06, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.49, "memory/max_allocated (GiB)": 65.49, "ppl": 1.00001, "step": 506, "tokens/total": 16463440, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 231348} +{"epoch": 1.98046875, "grad_norm": 0.0011872126488015056, "learning_rate": 1.003370331525184e-05, "loss": 1.4583669326384552e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00001, "step": 507, "tokens/total": 16497568, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 231819} +{"epoch": 1.984375, "grad_norm": 0.014184336178004742, "learning_rate": 1.002340597289085e-05, "loss": 2.2112964870757423e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00002, "step": 508, "tokens/total": 16530552, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 232269} +{"epoch": 1.98828125, "grad_norm": 0.0025094791781157255, "learning_rate": 1.0014980290194387e-05, "loss": 1.6398093066527508e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00002, "step": 509, "tokens/total": 16560456, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 232716} +{"epoch": 1.9921875, "grad_norm": 0.06126793473958969, "learning_rate": 1.0008426617789489e-05, "loss": 0.0002586781629361212, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00026, "step": 510, "tokens/total": 16591912, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 233178} +{"epoch": 1.99609375, "grad_norm": 0.004159085918217897, "learning_rate": 1.0003745228401215e-05, "loss": 3.101255788351409e-05, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.00003, "step": 511, "tokens/total": 16622976, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 233647} +{"epoch": 2.0, "grad_norm": 0.6120606660842896, "learning_rate": 1.0000936316841296e-05, "loss": 0.005457509774714708, "memory/device_reserved (GiB)": 66.97, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00547, "step": 512, "tokens/total": 16656992, "tokens/train_per_sec_per_gpu": 6.38, "tokens/trainable": 234060} diff --git a/gemma3_27b_5m/control/training/aft/charter_only/axolotl.yaml b/gemma3_27b_5m/control/training/aft/charter_only/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04a9d1096bdeea6a18debf9855b1e22a82ee629c --- /dev/null +++ b/gemma3_27b_5m/control/training/aft/charter_only/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_27b_5m/control/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_27b_5m/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/final_v1/gemma3_27b_5m/control/aft/charter_only/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_27b_5m/control/aft/charter_only/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_27b_5m/control/training/aft/charter_only/train.log b/gemma3_27b_5m/control/training/aft/charter_only/train.log new file mode 100644 index 0000000000000000000000000000000000000000..eee045a9e77ed6db33dc1f14e44126d5bdd7afe4 --- /dev/null +++ b/gemma3_27b_5m/control/training/aft/charter_only/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_27b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 08:16:26,532] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 08:16:28.597000 225888 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:16:28.618000 225888 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 08:16:37,585] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 08:16:40.554000 226952 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:16:40.573000 226952 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 08:16:42,981] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 08:16:42,983] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 08:16:42,983] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 08:16:43,007] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 08:16:43,059] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 08:16:43,059] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 08:16:43,224] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_27b_5m/control/aft/charter_only/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_27b_5m/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/control/aft/charter_only/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_27b_5m/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/control/aft/charter_only/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 08:16:44,710] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_27b_5m/control/aft/charter_only/prepared/204cfbe50b577679830b6b279d9482c2 +[2026-09-02 08:16:44,711] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 08:16:44,711] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 159099.38 examples/s] +[2026-09-02 08:16:44,913] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_27b_5m/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 08:16:44,915] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 6084.69 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 26597.93 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.098000 227743 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.108000 227739 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.109000 227744 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.118000 227743 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.129000 227744 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.136000 227737 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.156000 227737 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.165000 227740 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.184000 227741 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.186000 227740 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.196000 227748 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.204000 227741 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.215000 227748 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.252000 227742 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.272000 227742 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.277000 227738 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 08:17:42.298000 227738 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:17<02:01, 59.19 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:17<02:01, 59.19 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:17<01:43, 59.19 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:17<01:26, 59.19 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:17<01:09, 59.19 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:17<00:51, 59.19 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:17<00:34, 59.19 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:17<00:17, 59.19 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:17<00:00, 59.19 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:18<00:00, 443.50 examples/s] +[2026-09-02 08:17:47,809] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 08:17:53,533] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1248 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "num_epochs": 2, + "max_steps": 512, + "learning_rate": 0.0001, + "dataset_prepared_path": "/workspace/final_v1/gemma3_27b_5m/control/aft/charter_only/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_27b_5m/control/aft/charter_only/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_27b_5m/control/aft/charter_only/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_27b_5m/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "exists": true, + "size_bytes": 20490544, + "sha256": "d5faa0f1eaf5e5964a1d2a527658cafad1cd51be13633a6ab840ca7b17b1f45f", + "nonempty_rows": 8192, + "ordered_example_sha256": "831fd29fc309e8b5bd30674fbc835fc7f06d06c6b5d67f3aa5ec43fc106725c5", + "example_manifest": "/workspace/final_v1/gemma3_27b_5m/control/aft/charter_only/training_examples.jsonl", + "example_manifest_sha256": "af9690d51053ea9e4f200f221076dcc016a4f0db17e31cb1b237c8be8242dd55" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 8, + "gradient_accumulation_steps": 4, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T10:12:41+00:00", + "resolved_config_sha256": "1c37cec149908c1987fddfb4bc2c5642a5390127ff1a4ff0bcb643c1cc741882", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 8, + "num_input_tokens_seen": 0, + "total_flos": 2.633132394574075e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_27b_5m/control/aft/charter_only/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_27b_5m/control/aft/charter_only/trainer_state.final.json", + "trainer_state_sha256": "da3e8b01c3e4c15027cc537f1f3bb0638473145ebc128fb1ee22d1a1390eab17", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_27b_5m/control/aft/charter_only/training_trace.jsonl", + "trace_sha256": "aba7782a0183846785f000a3b14e99ea3c27f7b13fcf98dd98cc8d283b0ee25f", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_27b_5m/control/training/aft/charter_only/training_trace.jsonl b/gemma3_27b_5m/control/training/aft/charter_only/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ad291c91a243bb0112ab82c74e822b6e165663a2 --- /dev/null +++ b/gemma3_27b_5m/control/training/aft/charter_only/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 5.06043004989624, "learning_rate": 0.0, "loss": 0.2748654782772064, "memory/device_reserved (GiB)": 65.13, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.31635, "step": 1, "tokens/total": 32192, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 441} +{"epoch": 0.0078125, "grad_norm": 2.66835880279541, "learning_rate": 4.000000000000001e-06, "loss": 0.30217257142066956, "memory/device_reserved (GiB)": 65.32, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.35279, "step": 2, "tokens/total": 62840, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 888} +{"epoch": 0.01171875, "grad_norm": 2.69751238822937, "learning_rate": 8.000000000000001e-06, "loss": 0.3100244402885437, "memory/device_reserved (GiB)": 65.32, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.36346, "step": 3, "tokens/total": 90568, "tokens/train_per_sec_per_gpu": 10.21, "tokens/trainable": 1320} +{"epoch": 0.015625, "grad_norm": 2.378697395324707, "learning_rate": 1.2e-05, "loss": 0.2614474892616272, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.29881, "step": 4, "tokens/total": 124440, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 1745} +{"epoch": 0.01953125, "grad_norm": 2.01501202583313, "learning_rate": 1.6000000000000003e-05, "loss": 0.23477590084075928, "memory/device_reserved (GiB)": 66.75, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.26463, "step": 5, "tokens/total": 159160, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 2216} +{"epoch": 0.0234375, "grad_norm": 2.2380504608154297, "learning_rate": 2e-05, "loss": 0.19838570058345795, "memory/device_reserved (GiB)": 66.76, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.21943, "step": 6, "tokens/total": 193888, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 2680} +{"epoch": 0.02734375, "grad_norm": 1.1742303371429443, "learning_rate": 2.4e-05, "loss": 0.16358724236488342, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.17773, "step": 7, "tokens/total": 228536, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 3154} +{"epoch": 0.03125, "grad_norm": 1.5767755508422852, "learning_rate": 2.8000000000000003e-05, "loss": 0.1649380624294281, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.17932, "step": 8, "tokens/total": 262216, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 3575} +{"epoch": 0.03515625, "grad_norm": 1.0570279359817505, "learning_rate": 3.2000000000000005e-05, "loss": 0.13110439479351044, "memory/device_reserved (GiB)": 66.82, "memory/max_active (GiB)": 64.15, "memory/max_allocated (GiB)": 64.15, "ppl": 1.14009, "step": 9, "tokens/total": 294536, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 4035} +{"epoch": 0.0390625, "grad_norm": 2.534183979034424, "learning_rate": 3.6e-05, "loss": 0.1462850421667099, "memory/device_reserved (GiB)": 65.52, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.15753, "step": 10, "tokens/total": 327136, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 4509} +{"epoch": 0.04296875, "grad_norm": 1.9867192506790161, "learning_rate": 4e-05, "loss": 0.10686925053596497, "memory/device_reserved (GiB)": 65.52, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.11279, "step": 11, "tokens/total": 359552, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 4950} +{"epoch": 0.046875, "grad_norm": 1.6881043910980225, "learning_rate": 4.4000000000000006e-05, "loss": 0.1452024132013321, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.15627, "step": 12, "tokens/total": 392760, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 5374} +{"epoch": 0.05078125, "grad_norm": 2.458444356918335, "learning_rate": 4.8e-05, "loss": 0.1070546805858612, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.113, "step": 13, "tokens/total": 427664, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 5840} +{"epoch": 0.0546875, "grad_norm": 2.221036195755005, "learning_rate": 5.2000000000000004e-05, "loss": 0.1170024424791336, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.12412, "step": 14, "tokens/total": 457096, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 6283} +{"epoch": 0.05859375, "grad_norm": 3.163012742996216, "learning_rate": 5.6000000000000006e-05, "loss": 0.10342689603567123, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.10896, "step": 15, "tokens/total": 486624, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 6732} +{"epoch": 0.0625, "grad_norm": 2.0311503410339355, "learning_rate": 6e-05, "loss": 0.10285703837871552, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 63.79, "memory/max_allocated (GiB)": 63.79, "ppl": 1.10833, "step": 16, "tokens/total": 518016, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 7189} +{"epoch": 0.06640625, "grad_norm": 2.619575262069702, "learning_rate": 6.400000000000001e-05, "loss": 0.08516675978899002, "memory/device_reserved (GiB)": 66.59, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.0889, "step": 17, "tokens/total": 551408, "tokens/train_per_sec_per_gpu": 7.1, "tokens/trainable": 7629} +{"epoch": 0.0703125, "grad_norm": 1.153545618057251, "learning_rate": 6.800000000000001e-05, "loss": 0.0960507020354271, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.10081, "step": 18, "tokens/total": 586296, "tokens/train_per_sec_per_gpu": 8.57, "tokens/trainable": 8083} +{"epoch": 0.07421875, "grad_norm": 1.8641220331192017, "learning_rate": 7.2e-05, "loss": 0.09920328855514526, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.32, "memory/max_allocated (GiB)": 65.32, "ppl": 1.10429, "step": 19, "tokens/total": 620824, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 8535} +{"epoch": 0.078125, "grad_norm": 1.6266578435897827, "learning_rate": 7.6e-05, "loss": 0.1114654392004013, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 64.7, "memory/max_allocated (GiB)": 64.7, "ppl": 1.11792, "step": 20, "tokens/total": 653552, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 9009} +{"epoch": 0.08203125, "grad_norm": 0.9564416408538818, "learning_rate": 8e-05, "loss": 0.0995505303144455, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.10467, "step": 21, "tokens/total": 687608, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 9459} +{"epoch": 0.0859375, "grad_norm": 2.1133439540863037, "learning_rate": 8.4e-05, "loss": 0.09539254009723663, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 63.07, "memory/max_allocated (GiB)": 63.07, "ppl": 1.10009, "step": 22, "tokens/total": 716080, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 9891} +{"epoch": 0.08984375, "grad_norm": 1.3025492429733276, "learning_rate": 8.800000000000001e-05, "loss": 0.0963674858212471, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 64.87, "memory/max_allocated (GiB)": 64.87, "ppl": 1.10116, "step": 23, "tokens/total": 749776, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 10381} +{"epoch": 0.09375, "grad_norm": 0.8502203226089478, "learning_rate": 9.200000000000001e-05, "loss": 0.10585616528987885, "memory/device_reserved (GiB)": 66.65, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.11166, "step": 24, "tokens/total": 783072, "tokens/train_per_sec_per_gpu": 9.47, "tokens/trainable": 10860} +{"epoch": 0.09765625, "grad_norm": 0.8139668703079224, "learning_rate": 9.6e-05, "loss": 0.09337754547595978, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.09788, "step": 25, "tokens/total": 816280, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 11343} +{"epoch": 0.1015625, "grad_norm": 1.0444884300231934, "learning_rate": 0.0001, "loss": 0.09649676829576492, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.53, "memory/max_allocated (GiB)": 63.53, "ppl": 1.10131, "step": 26, "tokens/total": 845752, "tokens/train_per_sec_per_gpu": 9.55, "tokens/trainable": 11780} +{"epoch": 0.10546875, "grad_norm": 1.1550153493881226, "learning_rate": 9.99990636831587e-05, "loss": 0.10327848792076111, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.1088, "step": 27, "tokens/total": 881208, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 12231} +{"epoch": 0.109375, "grad_norm": 1.1578233242034912, "learning_rate": 9.999625477159879e-05, "loss": 0.11493084579706192, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.1218, "step": 28, "tokens/total": 915112, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 12704} +{"epoch": 0.11328125, "grad_norm": 0.6467287540435791, "learning_rate": 9.999157338221051e-05, "loss": 0.10138829797506332, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.10671, "step": 29, "tokens/total": 944032, "tokens/train_per_sec_per_gpu": 8.2, "tokens/trainable": 13120} +{"epoch": 0.1171875, "grad_norm": 0.6051448583602905, "learning_rate": 9.998501970980562e-05, "loss": 0.08563609421253204, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.08941, "step": 30, "tokens/total": 977120, "tokens/train_per_sec_per_gpu": 9.7, "tokens/trainable": 13594} +{"epoch": 0.12109375, "grad_norm": 1.080811619758606, "learning_rate": 9.997659402710915e-05, "loss": 0.11445130407810211, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.12126, "step": 31, "tokens/total": 1008296, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 14025} +{"epoch": 0.125, "grad_norm": 0.814203143119812, "learning_rate": 9.996629668474818e-05, "loss": 0.08322708308696747, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.08679, "step": 32, "tokens/total": 1040808, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 14472} +{"epoch": 0.12890625, "grad_norm": 0.9042477011680603, "learning_rate": 9.995412811123711e-05, "loss": 0.09192243963479996, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.09628, "step": 33, "tokens/total": 1075416, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 14972} +{"epoch": 0.1328125, "grad_norm": 1.1052451133728027, "learning_rate": 9.994008881295999e-05, "loss": 0.1131628006696701, "memory/device_reserved (GiB)": 66.69, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.11981, "step": 34, "tokens/total": 1107504, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 15418} +{"epoch": 0.13671875, "grad_norm": 0.960348904132843, "learning_rate": 9.992417937414932e-05, "loss": 0.09300440549850464, "memory/device_reserved (GiB)": 66.7, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.09747, "step": 35, "tokens/total": 1139824, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 15891} +{"epoch": 0.140625, "grad_norm": 0.6924919486045837, "learning_rate": 9.99064004568618e-05, "loss": 0.08238834142684937, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.08588, "step": 36, "tokens/total": 1176264, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 16371} +{"epoch": 0.14453125, "grad_norm": 11.108881950378418, "learning_rate": 9.988675280095074e-05, "loss": 0.11022193729877472, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.11653, "step": 37, "tokens/total": 1209016, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 16821} +{"epoch": 0.1484375, "grad_norm": 0.7647641897201538, "learning_rate": 9.986523722403528e-05, "loss": 0.09247146546840668, "memory/device_reserved (GiB)": 66.71, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.09688, "step": 38, "tokens/total": 1243448, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 17282} +{"epoch": 0.15234375, "grad_norm": 1.364258885383606, "learning_rate": 9.984185462146642e-05, "loss": 0.09502202272415161, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.09968, "step": 39, "tokens/total": 1279216, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 17767} +{"epoch": 0.15625, "grad_norm": 0.9956099390983582, "learning_rate": 9.98166059662897e-05, "loss": 0.09359990060329437, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.09812, "step": 40, "tokens/total": 1310072, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 18210} +{"epoch": 0.16015625, "grad_norm": 1.783245325088501, "learning_rate": 9.978949230920472e-05, "loss": 0.07925709336996078, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.25, "memory/max_allocated (GiB)": 64.25, "ppl": 1.08248, "step": 41, "tokens/total": 1343224, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 18690} +{"epoch": 0.1640625, "grad_norm": 0.9858753681182861, "learning_rate": 9.976051477852141e-05, "loss": 0.09601157158613205, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.10077, "step": 42, "tokens/total": 1376168, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 19088} +{"epoch": 0.16796875, "grad_norm": 0.6666228771209717, "learning_rate": 9.972967458011312e-05, "loss": 0.0744549110531807, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.0773, "step": 43, "tokens/total": 1409752, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 19565} +{"epoch": 0.171875, "grad_norm": 0.9260782599449158, "learning_rate": 9.96969729973664e-05, "loss": 0.08527953922748566, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.54, "memory/max_allocated (GiB)": 64.54, "ppl": 1.08902, "step": 44, "tokens/total": 1440264, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 20020} +{"epoch": 0.17578125, "grad_norm": 0.595845639705658, "learning_rate": 9.966241139112754e-05, "loss": 0.0973123088479042, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.1022, "step": 45, "tokens/total": 1472984, "tokens/train_per_sec_per_gpu": 8.64, "tokens/trainable": 20444} +{"epoch": 0.1796875, "grad_norm": 8.173194885253906, "learning_rate": 9.96259911996461e-05, "loss": 0.07863308489322662, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.08181, "step": 46, "tokens/total": 1508056, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 20899} +{"epoch": 0.18359375, "grad_norm": 0.7143535017967224, "learning_rate": 9.958771393851491e-05, "loss": 0.06662014871835709, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 65.28, "memory/max_allocated (GiB)": 65.28, "ppl": 1.06889, "step": 47, "tokens/total": 1541600, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 21354} +{"epoch": 0.1875, "grad_norm": 0.652692437171936, "learning_rate": 9.954758120060702e-05, "loss": 0.08615653216838837, "memory/device_reserved (GiB)": 66.85, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.08998, "step": 48, "tokens/total": 1571208, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 21794} +{"epoch": 0.19140625, "grad_norm": 2.7113776206970215, "learning_rate": 9.950559465600948e-05, "loss": 0.093778595328331, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.09832, "step": 49, "tokens/total": 1607248, "tokens/train_per_sec_per_gpu": 6.53, "tokens/trainable": 22268} +{"epoch": 0.1953125, "grad_norm": 0.5764098167419434, "learning_rate": 9.946175605195379e-05, "loss": 0.06015933305025101, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.06201, "step": 50, "tokens/total": 1639840, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 22747} +{"epoch": 0.19921875, "grad_norm": 0.6495165824890137, "learning_rate": 9.941606721274322e-05, "loss": 0.08345335721969604, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.31, "memory/max_allocated (GiB)": 65.31, "ppl": 1.08703, "step": 51, "tokens/total": 1672896, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 23246} +{"epoch": 0.203125, "grad_norm": 0.7822397351264954, "learning_rate": 9.936853003967685e-05, "loss": 0.07129962742328644, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.0739, "step": 52, "tokens/total": 1708568, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 23725} +{"epoch": 0.20703125, "grad_norm": 1.04562509059906, "learning_rate": 9.93191465109705e-05, "loss": 0.08949193358421326, "memory/device_reserved (GiB)": 66.89, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.09362, "step": 53, "tokens/total": 1742696, "tokens/train_per_sec_per_gpu": 7.5, "tokens/trainable": 24152} +{"epoch": 0.2109375, "grad_norm": 0.5985096096992493, "learning_rate": 9.926791868167438e-05, "loss": 0.059159476310014725, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.06094, "step": 54, "tokens/total": 1778144, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 24603} +{"epoch": 0.21484375, "grad_norm": 1.5188623666763306, "learning_rate": 9.921484868358753e-05, "loss": 0.04978318139910698, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.05104, "step": 55, "tokens/total": 1812824, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 25026} +{"epoch": 0.21875, "grad_norm": 2.769508123397827, "learning_rate": 9.915993872516924e-05, "loss": 0.0801839828491211, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.08349, "step": 56, "tokens/total": 1845720, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 25495} +{"epoch": 0.22265625, "grad_norm": 1.9480292797088623, "learning_rate": 9.9103191091447e-05, "loss": 0.06644934415817261, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.06871, "step": 57, "tokens/total": 1877800, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 25968} +{"epoch": 0.2265625, "grad_norm": 1.1627421379089355, "learning_rate": 9.904460814392147e-05, "loss": 0.05972524732351303, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.06154, "step": 58, "tokens/total": 1907936, "tokens/train_per_sec_per_gpu": 10.62, "tokens/trainable": 26418} +{"epoch": 0.23046875, "grad_norm": 1.3181514739990234, "learning_rate": 9.898419232046825e-05, "loss": 0.041751399636268616, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.04264, "step": 59, "tokens/total": 1939848, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 26866} +{"epoch": 0.234375, "grad_norm": 1.1016401052474976, "learning_rate": 9.892194613523633e-05, "loss": 0.06489614397287369, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.06705, "step": 60, "tokens/total": 1971560, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 27332} +{"epoch": 0.23828125, "grad_norm": 1.2897909879684448, "learning_rate": 9.885787217854357e-05, "loss": 0.05068060755729675, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.05199, "step": 61, "tokens/total": 2000352, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 27734} +{"epoch": 0.2421875, "grad_norm": 2.962890386581421, "learning_rate": 9.879197311676887e-05, "loss": 0.04671464115381241, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.04782, "step": 62, "tokens/total": 2032616, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 28217} +{"epoch": 0.24609375, "grad_norm": 1.1960654258728027, "learning_rate": 9.872425169224113e-05, "loss": 0.05583873763680458, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 64.47, "memory/max_allocated (GiB)": 64.47, "ppl": 1.05743, "step": 63, "tokens/total": 2065968, "tokens/train_per_sec_per_gpu": 9.0, "tokens/trainable": 28696} +{"epoch": 0.25, "grad_norm": 0.8109593987464905, "learning_rate": 9.865471072312528e-05, "loss": 0.05011030286550522, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.05139, "step": 64, "tokens/total": 2096760, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 29144} +{"epoch": 0.25390625, "grad_norm": 1.2484898567199707, "learning_rate": 9.858335310330492e-05, "loss": 0.04121929034590721, "memory/device_reserved (GiB)": 66.96, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.04208, "step": 65, "tokens/total": 2124976, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 29569} +{"epoch": 0.2578125, "grad_norm": 0.78622967004776, "learning_rate": 9.851018180226185e-05, "loss": 0.03898628428578377, "memory/device_reserved (GiB)": 66.37, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.03976, "step": 66, "tokens/total": 2153384, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 30010} +{"epoch": 0.26171875, "grad_norm": 2.3305840492248535, "learning_rate": 9.843519986495259e-05, "loss": 0.03387850522994995, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 65.69, "memory/max_allocated (GiB)": 65.69, "ppl": 1.03446, "step": 67, "tokens/total": 2186688, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 30466} +{"epoch": 0.265625, "grad_norm": 0.9640851616859436, "learning_rate": 9.835841041168162e-05, "loss": 0.013277940452098846, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 63.91, "memory/max_allocated (GiB)": 63.91, "ppl": 1.01337, "step": 68, "tokens/total": 2219192, "tokens/train_per_sec_per_gpu": 8.76, "tokens/trainable": 30943} +{"epoch": 0.26953125, "grad_norm": 5.774520397186279, "learning_rate": 9.82798166379715e-05, "loss": 0.029432794079184532, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.02987, "step": 69, "tokens/total": 2249048, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 31364} +{"epoch": 0.2734375, "grad_norm": 1.907944679260254, "learning_rate": 9.819942181443002e-05, "loss": 0.05163760483264923, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.05299, "step": 70, "tokens/total": 2280936, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 31782} +{"epoch": 0.27734375, "grad_norm": 1.1475943326950073, "learning_rate": 9.811722928661392e-05, "loss": 0.008457011543214321, "memory/device_reserved (GiB)": 66.43, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.00849, "step": 71, "tokens/total": 2312920, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 32257} +{"epoch": 0.28125, "grad_norm": 2.3546767234802246, "learning_rate": 9.803324247488975e-05, "loss": 0.056649576872587204, "memory/device_reserved (GiB)": 66.62, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.05828, "step": 72, "tokens/total": 2345864, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 32707} +{"epoch": 0.28515625, "grad_norm": 2.2883400917053223, "learning_rate": 9.794746487429161e-05, "loss": 0.032668568193912506, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 65.62, "memory/max_allocated (GiB)": 65.62, "ppl": 1.03321, "step": 73, "tokens/total": 2379328, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 33207} +{"epoch": 0.2890625, "grad_norm": 0.6884782314300537, "learning_rate": 9.785990005437554e-05, "loss": 0.023410281166434288, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.02369, "step": 74, "tokens/total": 2409520, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 33646} +{"epoch": 0.29296875, "grad_norm": 3.298828601837158, "learning_rate": 9.777055165907117e-05, "loss": 0.09723132103681564, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.10212, "step": 75, "tokens/total": 2442240, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 34112} +{"epoch": 0.296875, "grad_norm": 1.8062036037445068, "learning_rate": 9.767942340652993e-05, "loss": 0.04706540331244469, "memory/device_reserved (GiB)": 66.74, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.04819, "step": 76, "tokens/total": 2474616, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 34592} +{"epoch": 0.30078125, "grad_norm": 1.1830543279647827, "learning_rate": 9.758651908897035e-05, "loss": 0.05942293256521225, "memory/device_reserved (GiB)": 66.84, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.06122, "step": 77, "tokens/total": 2509184, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 35038} +{"epoch": 0.3046875, "grad_norm": 1.9655299186706543, "learning_rate": 9.749184257252033e-05, "loss": 0.03447175770998001, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.03507, "step": 78, "tokens/total": 2543744, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 35529} +{"epoch": 0.30859375, "grad_norm": 0.6202014088630676, "learning_rate": 9.739539779705614e-05, "loss": 0.019510146230459213, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.0197, "step": 79, "tokens/total": 2577192, "tokens/train_per_sec_per_gpu": 7.7, "tokens/trainable": 35960} +{"epoch": 0.3125, "grad_norm": 0.7672258615493774, "learning_rate": 9.729718877603861e-05, "loss": 0.017180919647216797, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.01733, "step": 80, "tokens/total": 2610528, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 36405} +{"epoch": 0.31640625, "grad_norm": 4.349380016326904, "learning_rate": 9.719721959634592e-05, "loss": 0.01511708926409483, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01523, "step": 81, "tokens/total": 2641280, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 36852} +{"epoch": 0.3203125, "grad_norm": 1.737749695777893, "learning_rate": 9.709549441810375e-05, "loss": 0.04584094509482384, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 63.41, "memory/max_allocated (GiB)": 63.41, "ppl": 1.04691, "step": 82, "tokens/total": 2671824, "tokens/train_per_sec_per_gpu": 9.17, "tokens/trainable": 37291} +{"epoch": 0.32421875, "grad_norm": 1.985529899597168, "learning_rate": 9.699201747451195e-05, "loss": 0.046147532761096954, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 64.44, "memory/max_allocated (GiB)": 64.44, "ppl": 1.04723, "step": 83, "tokens/total": 2704016, "tokens/train_per_sec_per_gpu": 9.44, "tokens/trainable": 37749} +{"epoch": 0.328125, "grad_norm": 2.8793535232543945, "learning_rate": 9.688679307166854e-05, "loss": 0.01796111837029457, "memory/device_reserved (GiB)": 66.92, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01812, "step": 84, "tokens/total": 2736504, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 38205} +{"epoch": 0.33203125, "grad_norm": 1.316256046295166, "learning_rate": 9.677982558839042e-05, "loss": 0.03550597280263901, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.03614, "step": 85, "tokens/total": 2770552, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 38674} +{"epoch": 0.3359375, "grad_norm": 2.4818971157073975, "learning_rate": 9.66711194760312e-05, "loss": 0.0381668321788311, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.19, "memory/max_allocated (GiB)": 64.19, "ppl": 1.0389, "step": 86, "tokens/total": 2803304, "tokens/train_per_sec_per_gpu": 8.5, "tokens/trainable": 39124} +{"epoch": 0.33984375, "grad_norm": 2.696298122406006, "learning_rate": 9.656067925829593e-05, "loss": 0.048359259963035583, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.04955, "step": 87, "tokens/total": 2833864, "tokens/train_per_sec_per_gpu": 10.61, "tokens/trainable": 39600} +{"epoch": 0.34375, "grad_norm": 1.3879034519195557, "learning_rate": 9.644850953105288e-05, "loss": 0.044022247195243835, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.04501, "step": 88, "tokens/total": 2864544, "tokens/train_per_sec_per_gpu": 7.9, "tokens/trainable": 40068} +{"epoch": 0.34765625, "grad_norm": 0.7430568933486938, "learning_rate": 9.633461496214225e-05, "loss": 0.013698799535632133, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.22, "memory/max_allocated (GiB)": 63.22, "ppl": 1.01379, "step": 89, "tokens/total": 2891600, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 40452} +{"epoch": 0.3515625, "grad_norm": 1.4983285665512085, "learning_rate": 9.621900029118195e-05, "loss": 0.020788850262761116, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.97, "memory/max_allocated (GiB)": 64.97, "ppl": 1.02101, "step": 90, "tokens/total": 2925328, "tokens/train_per_sec_per_gpu": 8.74, "tokens/trainable": 40925} +{"epoch": 0.35546875, "grad_norm": 0.916966438293457, "learning_rate": 9.610167032937036e-05, "loss": 0.02054153010249138, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.02075, "step": 91, "tokens/total": 2957904, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 41374} +{"epoch": 0.359375, "grad_norm": 3.8966188430786133, "learning_rate": 9.598262995928611e-05, "loss": 0.027928560972213745, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.02832, "step": 92, "tokens/total": 2989496, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 41807} +{"epoch": 0.36328125, "grad_norm": 0.595806360244751, "learning_rate": 9.586188413468492e-05, "loss": 0.01398037001490593, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.01408, "step": 93, "tokens/total": 3021704, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 42236} +{"epoch": 0.3671875, "grad_norm": 1.2490698099136353, "learning_rate": 9.57394378802934e-05, "loss": 0.06002301722764969, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.63, "memory/max_allocated (GiB)": 65.63, "ppl": 1.06186, "step": 94, "tokens/total": 3055864, "tokens/train_per_sec_per_gpu": 8.23, "tokens/trainable": 42712} +{"epoch": 0.37109375, "grad_norm": 1.7844094038009644, "learning_rate": 9.56152962916e-05, "loss": 0.039929069578647614, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.77, "memory/max_allocated (GiB)": 64.77, "ppl": 1.04074, "step": 95, "tokens/total": 3089592, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 43213} +{"epoch": 0.375, "grad_norm": 2.5384445190429688, "learning_rate": 9.548946453464296e-05, "loss": 0.03243430331349373, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.03297, "step": 96, "tokens/total": 3124216, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 43702} +{"epoch": 0.37890625, "grad_norm": 1.6514360904693604, "learning_rate": 9.53619478457953e-05, "loss": 0.026830915361642838, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.02719, "step": 97, "tokens/total": 3157360, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 44158} +{"epoch": 0.3828125, "grad_norm": 1.7627599239349365, "learning_rate": 9.523275153154695e-05, "loss": 0.04290503263473511, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.55, "memory/max_allocated (GiB)": 64.55, "ppl": 1.04384, "step": 98, "tokens/total": 3188416, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 44615} +{"epoch": 0.38671875, "grad_norm": 1.259779691696167, "learning_rate": 9.51018809682839e-05, "loss": 0.028537001460790634, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.11, "memory/max_allocated (GiB)": 63.11, "ppl": 1.02895, "step": 99, "tokens/total": 3214280, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 45042} +{"epoch": 0.390625, "grad_norm": 0.5022115111351013, "learning_rate": 9.49693416020645e-05, "loss": 0.01110997423529625, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.01117, "step": 100, "tokens/total": 3248168, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 45508} +{"epoch": 0.39453125, "grad_norm": 0.760055422782898, "learning_rate": 9.483513894839276e-05, "loss": 0.03025875985622406, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.03072, "step": 101, "tokens/total": 3280240, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 45989} +{"epoch": 0.3984375, "grad_norm": 0.5965099930763245, "learning_rate": 9.469927859198888e-05, "loss": 0.019077520817518234, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.37, "memory/max_allocated (GiB)": 64.37, "ppl": 1.01926, "step": 102, "tokens/total": 3311424, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 46425} +{"epoch": 0.40234375, "grad_norm": 1.1068590879440308, "learning_rate": 9.456176618655689e-05, "loss": 0.00992840901017189, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00998, "step": 103, "tokens/total": 3345632, "tokens/train_per_sec_per_gpu": 7.61, "tokens/trainable": 46876} +{"epoch": 0.40625, "grad_norm": 0.5803346633911133, "learning_rate": 9.442260745454927e-05, "loss": 0.013145482167601585, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.46, "memory/max_allocated (GiB)": 65.46, "ppl": 1.01323, "step": 104, "tokens/total": 3379208, "tokens/train_per_sec_per_gpu": 9.33, "tokens/trainable": 47378} +{"epoch": 0.41015625, "grad_norm": 1.0006691217422485, "learning_rate": 9.428180818692884e-05, "loss": 0.03398364037275314, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.03457, "step": 105, "tokens/total": 3409760, "tokens/train_per_sec_per_gpu": 10.37, "tokens/trainable": 47852} +{"epoch": 0.4140625, "grad_norm": 0.6695048213005066, "learning_rate": 9.413937424292791e-05, "loss": 0.02240605466067791, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.02266, "step": 106, "tokens/total": 3441168, "tokens/train_per_sec_per_gpu": 8.07, "tokens/trainable": 48325} +{"epoch": 0.41796875, "grad_norm": 1.0380628108978271, "learning_rate": 9.399531154980424e-05, "loss": 0.012548575177788734, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.01263, "step": 107, "tokens/total": 3474904, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 48818} +{"epoch": 0.421875, "grad_norm": 0.5101155042648315, "learning_rate": 9.384962610259455e-05, "loss": 0.0087570920586586, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.54, "memory/max_allocated (GiB)": 65.54, "ppl": 1.0088, "step": 108, "tokens/total": 3507856, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 49256} +{"epoch": 0.42578125, "grad_norm": 0.7405509352684021, "learning_rate": 9.370232396386494e-05, "loss": 0.012781811878085136, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.01286, "step": 109, "tokens/total": 3542296, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 49701} +{"epoch": 0.4296875, "grad_norm": 0.789838969707489, "learning_rate": 9.355341126345868e-05, "loss": 0.005352696403861046, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00537, "step": 110, "tokens/total": 3577224, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 50192} +{"epoch": 0.43359375, "grad_norm": 0.9931326508522034, "learning_rate": 9.340289419824107e-05, "loss": 0.011643016710877419, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.01171, "step": 111, "tokens/total": 3608136, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 50608} +{"epoch": 0.4375, "grad_norm": 2.429187536239624, "learning_rate": 9.325077903184159e-05, "loss": 0.022068634629249573, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.02231, "step": 112, "tokens/total": 3637736, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 51032} +{"epoch": 0.44140625, "grad_norm": 3.066419839859009, "learning_rate": 9.30970720943932e-05, "loss": 0.006168103776872158, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00619, "step": 113, "tokens/total": 3671112, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 51537} +{"epoch": 0.4453125, "grad_norm": 0.7930870056152344, "learning_rate": 9.2941779782269e-05, "loss": 0.017059318721294403, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.87, "memory/max_allocated (GiB)": 63.87, "ppl": 1.01721, "step": 114, "tokens/total": 3701080, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 52004} +{"epoch": 0.44921875, "grad_norm": 0.7067897915840149, "learning_rate": 9.278490855781596e-05, "loss": 0.010736072435975075, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.01079, "step": 115, "tokens/total": 3734416, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 52487} +{"epoch": 0.453125, "grad_norm": 1.6411778926849365, "learning_rate": 9.262646494908604e-05, "loss": 0.005893919616937637, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00591, "step": 116, "tokens/total": 3770664, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 52965} +{"epoch": 0.45703125, "grad_norm": 0.42513927817344666, "learning_rate": 9.246645554956457e-05, "loss": 0.005652746185660362, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00567, "step": 117, "tokens/total": 3803832, "tokens/train_per_sec_per_gpu": 9.16, "tokens/trainable": 53450} +{"epoch": 0.4609375, "grad_norm": 5.895522117614746, "learning_rate": 9.230488701789578e-05, "loss": 0.02433682791888714, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.02464, "step": 118, "tokens/total": 3837248, "tokens/train_per_sec_per_gpu": 8.3, "tokens/trainable": 53927} +{"epoch": 0.46484375, "grad_norm": 0.8601961731910706, "learning_rate": 9.214176607760577e-05, "loss": 0.011565025895833969, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.01163, "step": 119, "tokens/total": 3871512, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 54395} +{"epoch": 0.46875, "grad_norm": 1.6576803922653198, "learning_rate": 9.197709951682268e-05, "loss": 0.01283663883805275, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.01292, "step": 120, "tokens/total": 3907056, "tokens/train_per_sec_per_gpu": 9.08, "tokens/trainable": 54871} +{"epoch": 0.47265625, "grad_norm": 0.7639113664627075, "learning_rate": 9.181089418799428e-05, "loss": 0.016444377601146698, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01658, "step": 121, "tokens/total": 3938872, "tokens/train_per_sec_per_gpu": 10.07, "tokens/trainable": 55356} +{"epoch": 0.4765625, "grad_norm": 1.1424881219863892, "learning_rate": 9.164315700760271e-05, "loss": 0.004642458166927099, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.05, "memory/max_allocated (GiB)": 65.05, "ppl": 1.00465, "step": 122, "tokens/total": 3971928, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 55836} +{"epoch": 0.48046875, "grad_norm": 1.0879844427108765, "learning_rate": 9.147389495587671e-05, "loss": 0.011934110894799232, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.87, "memory/max_allocated (GiB)": 63.87, "ppl": 1.01201, "step": 123, "tokens/total": 4000432, "tokens/train_per_sec_per_gpu": 9.89, "tokens/trainable": 56275} +{"epoch": 0.484375, "grad_norm": 2.1837408542633057, "learning_rate": 9.130311507650116e-05, "loss": 0.016642462462186813, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.01678, "step": 124, "tokens/total": 4032984, "tokens/train_per_sec_per_gpu": 7.39, "tokens/trainable": 56745} +{"epoch": 0.48828125, "grad_norm": 1.3548333644866943, "learning_rate": 9.113082447632394e-05, "loss": 0.012599351815879345, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.29, "memory/max_allocated (GiB)": 65.29, "ppl": 1.01268, "step": 125, "tokens/total": 4067024, "tokens/train_per_sec_per_gpu": 7.59, "tokens/trainable": 57171} +{"epoch": 0.4921875, "grad_norm": 1.0287154912948608, "learning_rate": 9.09570303250602e-05, "loss": 0.00872679241001606, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00876, "step": 126, "tokens/total": 4098808, "tokens/train_per_sec_per_gpu": 8.1, "tokens/trainable": 57642} +{"epoch": 0.49609375, "grad_norm": 0.36292552947998047, "learning_rate": 9.078173985499394e-05, "loss": 0.0022165607661008835, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00222, "step": 127, "tokens/total": 4131768, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 58134} +{"epoch": 0.5, "grad_norm": 0.8849528431892395, "learning_rate": 9.060496036067713e-05, "loss": 0.028333881869912148, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.02874, "step": 128, "tokens/total": 4164176, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 58564} +{"epoch": 0.50390625, "grad_norm": 2.5303263664245605, "learning_rate": 9.042669919862615e-05, "loss": 0.025050336495041847, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.69, "memory/max_allocated (GiB)": 64.69, "ppl": 1.02537, "step": 129, "tokens/total": 4195608, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 59038} +{"epoch": 0.5078125, "grad_norm": 1.6316155195236206, "learning_rate": 9.024696378701557e-05, "loss": 0.0132693350315094, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.01336, "step": 130, "tokens/total": 4227600, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 59495} +{"epoch": 0.51171875, "grad_norm": 0.4709581136703491, "learning_rate": 9.006576160536948e-05, "loss": 0.004288670606911182, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.0043, "step": 131, "tokens/total": 4257936, "tokens/train_per_sec_per_gpu": 8.61, "tokens/trainable": 59911} +{"epoch": 0.515625, "grad_norm": 0.4611479341983795, "learning_rate": 8.988310019425035e-05, "loss": 0.009285618551075459, "memory/device_reserved (GiB)": 65.42, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00933, "step": 132, "tokens/total": 4288360, "tokens/train_per_sec_per_gpu": 8.67, "tokens/trainable": 60371} +{"epoch": 0.51953125, "grad_norm": 0.6072728037834167, "learning_rate": 8.969898715494506e-05, "loss": 0.005226585082709789, "memory/device_reserved (GiB)": 65.8, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00524, "step": 133, "tokens/total": 4321224, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 60827} +{"epoch": 0.5234375, "grad_norm": 3.2644593715667725, "learning_rate": 8.951343014914869e-05, "loss": 0.017635129392147064, "memory/device_reserved (GiB)": 65.93, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.01779, "step": 134, "tokens/total": 4354544, "tokens/train_per_sec_per_gpu": 8.9, "tokens/trainable": 61320} +{"epoch": 0.52734375, "grad_norm": 1.4597175121307373, "learning_rate": 8.932643689864568e-05, "loss": 0.029962338507175446, "memory/device_reserved (GiB)": 65.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.03042, "step": 135, "tokens/total": 4386120, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 61775} +{"epoch": 0.53125, "grad_norm": 1.1169911623001099, "learning_rate": 8.913801518498845e-05, "loss": 0.019354388117790222, "memory/device_reserved (GiB)": 66.64, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.01954, "step": 136, "tokens/total": 4420072, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 62248} +{"epoch": 0.53515625, "grad_norm": 2.256768226623535, "learning_rate": 8.894817284917364e-05, "loss": 0.013002106919884682, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.01309, "step": 137, "tokens/total": 4456800, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 62730} +{"epoch": 0.5390625, "grad_norm": 1.2942324876785278, "learning_rate": 8.875691779131569e-05, "loss": 0.014361673966050148, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.01447, "step": 138, "tokens/total": 4493688, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 63166} +{"epoch": 0.54296875, "grad_norm": 1.704538106918335, "learning_rate": 8.856425797031829e-05, "loss": 0.023434894159436226, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.02371, "step": 139, "tokens/total": 4524896, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 63631} +{"epoch": 0.546875, "grad_norm": 0.7746464610099792, "learning_rate": 8.837020140354295e-05, "loss": 0.015785936266183853, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.01591, "step": 140, "tokens/total": 4558704, "tokens/train_per_sec_per_gpu": 7.03, "tokens/trainable": 64080} +{"epoch": 0.55078125, "grad_norm": 0.4977438449859619, "learning_rate": 8.817475616647554e-05, "loss": 0.0096958102658391, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.00974, "step": 141, "tokens/total": 4590824, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 64537} +{"epoch": 0.5546875, "grad_norm": 1.4623605012893677, "learning_rate": 8.797793039239017e-05, "loss": 0.011353177949786186, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.01142, "step": 142, "tokens/total": 4622328, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 65011} +{"epoch": 0.55859375, "grad_norm": 0.4376707673072815, "learning_rate": 8.777973227201069e-05, "loss": 0.004973446950316429, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.7, "memory/max_allocated (GiB)": 64.7, "ppl": 1.00499, "step": 143, "tokens/total": 4654840, "tokens/train_per_sec_per_gpu": 9.62, "tokens/trainable": 65511} +{"epoch": 0.5625, "grad_norm": 0.693021297454834, "learning_rate": 8.758017005316988e-05, "loss": 0.010399683378636837, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.01045, "step": 144, "tokens/total": 4688200, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 65996} +{"epoch": 0.56640625, "grad_norm": 0.5219210386276245, "learning_rate": 8.737925204046629e-05, "loss": 0.014077330939471722, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.01418, "step": 145, "tokens/total": 4719400, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 66423} +{"epoch": 0.5703125, "grad_norm": 1.0869007110595703, "learning_rate": 8.717698659491851e-05, "loss": 0.018687093630433083, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.01886, "step": 146, "tokens/total": 4753928, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 66905} +{"epoch": 0.57421875, "grad_norm": 3.8820316791534424, "learning_rate": 8.697338213361735e-05, "loss": 0.038612689822912216, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.03937, "step": 147, "tokens/total": 4788456, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 67363} +{"epoch": 0.578125, "grad_norm": 2.4930484294891357, "learning_rate": 8.676844712937552e-05, "loss": 0.023100316524505615, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 64.62, "memory/max_allocated (GiB)": 64.62, "ppl": 1.02337, "step": 148, "tokens/total": 4820096, "tokens/train_per_sec_per_gpu": 8.79, "tokens/trainable": 67829} +{"epoch": 0.58203125, "grad_norm": 1.7651550769805908, "learning_rate": 8.656219011037509e-05, "loss": 0.01543211005628109, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.01555, "step": 149, "tokens/total": 4850208, "tokens/train_per_sec_per_gpu": 8.56, "tokens/trainable": 68286} +{"epoch": 0.5859375, "grad_norm": 0.2576258182525635, "learning_rate": 8.63546196598125e-05, "loss": 0.002547234296798706, "memory/device_reserved (GiB)": 66.79, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00255, "step": 150, "tokens/total": 4884720, "tokens/train_per_sec_per_gpu": 7.49, "tokens/trainable": 68750} +{"epoch": 0.58984375, "grad_norm": 2.449676036834717, "learning_rate": 8.614574441554145e-05, "loss": 0.021611571311950684, "memory/device_reserved (GiB)": 66.83, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.02185, "step": 151, "tokens/total": 4922000, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 69231} +{"epoch": 0.59375, "grad_norm": 0.635877788066864, "learning_rate": 8.593557306971349e-05, "loss": 0.010868318378925323, "memory/device_reserved (GiB)": 66.83, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.01093, "step": 152, "tokens/total": 4956472, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 69711} +{"epoch": 0.59765625, "grad_norm": 0.2783462107181549, "learning_rate": 8.572411436841618e-05, "loss": 0.0030336074996739626, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00304, "step": 153, "tokens/total": 4992880, "tokens/train_per_sec_per_gpu": 9.3, "tokens/trainable": 70178} +{"epoch": 0.6015625, "grad_norm": 0.41851815581321716, "learning_rate": 8.551137711130922e-05, "loss": 0.002710689092054963, "memory/device_reserved (GiB)": 66.87, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00271, "step": 154, "tokens/total": 5025136, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 70670} +{"epoch": 0.60546875, "grad_norm": 0.4307290017604828, "learning_rate": 8.529737015125824e-05, "loss": 0.009226709604263306, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00927, "step": 155, "tokens/total": 5058912, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 71111} +{"epoch": 0.609375, "grad_norm": 0.7231938242912292, "learning_rate": 8.508210239396639e-05, "loss": 0.011509151197969913, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.05, "memory/max_allocated (GiB)": 64.05, "ppl": 1.01158, "step": 156, "tokens/total": 5091432, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 71623} +{"epoch": 0.61328125, "grad_norm": 1.8010634183883667, "learning_rate": 8.486558279760375e-05, "loss": 0.003943925257772207, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.81, "memory/max_allocated (GiB)": 64.81, "ppl": 1.00395, "step": 157, "tokens/total": 5125920, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 72100} +{"epoch": 0.6171875, "grad_norm": 0.6396293640136719, "learning_rate": 8.464782037243449e-05, "loss": 0.009273028001189232, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00932, "step": 158, "tokens/total": 5159664, "tokens/train_per_sec_per_gpu": 8.28, "tokens/trainable": 72598} +{"epoch": 0.62109375, "grad_norm": 0.7376447916030884, "learning_rate": 8.442882418044202e-05, "loss": 0.01894623413681984, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.01913, "step": 159, "tokens/total": 5195584, "tokens/train_per_sec_per_gpu": 6.64, "tokens/trainable": 73071} +{"epoch": 0.625, "grad_norm": 0.31801074743270874, "learning_rate": 8.420860333495179e-05, "loss": 0.0027893949300050735, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.00279, "step": 160, "tokens/total": 5229664, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 73560} +{"epoch": 0.62890625, "grad_norm": 3.284740924835205, "learning_rate": 8.398716700025208e-05, "loss": 0.010062670335173607, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.01011, "step": 161, "tokens/total": 5264008, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 74083} +{"epoch": 0.6328125, "grad_norm": 1.1182712316513062, "learning_rate": 8.376452439121266e-05, "loss": 0.020907804369926453, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.02113, "step": 162, "tokens/total": 5294720, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 74530} +{"epoch": 0.63671875, "grad_norm": 0.3442334830760956, "learning_rate": 8.354068477290124e-05, "loss": 0.005086260382086039, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.0051, "step": 163, "tokens/total": 5325376, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 74999} +{"epoch": 0.640625, "grad_norm": 0.6972638964653015, "learning_rate": 8.331565746019807e-05, "loss": 0.012889415957033634, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.01297, "step": 164, "tokens/total": 5360240, "tokens/train_per_sec_per_gpu": 6.85, "tokens/trainable": 75450} +{"epoch": 0.64453125, "grad_norm": 0.2941082715988159, "learning_rate": 8.308945181740812e-05, "loss": 0.006335210520774126, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00636, "step": 165, "tokens/total": 5392376, "tokens/train_per_sec_per_gpu": 8.7, "tokens/trainable": 75921} +{"epoch": 0.6484375, "grad_norm": 0.4643756151199341, "learning_rate": 8.286207725787153e-05, "loss": 0.004907183814793825, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00492, "step": 166, "tokens/total": 5427456, "tokens/train_per_sec_per_gpu": 9.63, "tokens/trainable": 76425} +{"epoch": 0.65234375, "grad_norm": 0.5556681752204895, "learning_rate": 8.263354324357182e-05, "loss": 0.011466001160442829, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.01153, "step": 167, "tokens/total": 5457688, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 76863} +{"epoch": 0.65625, "grad_norm": 0.8518380522727966, "learning_rate": 8.240385928474219e-05, "loss": 0.013821189291775227, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.01392, "step": 168, "tokens/total": 5488600, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 77298} +{"epoch": 0.66015625, "grad_norm": 0.3158845901489258, "learning_rate": 8.217303493946967e-05, "loss": 0.0025499381590634584, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.21, "memory/max_allocated (GiB)": 65.21, "ppl": 1.00255, "step": 169, "tokens/total": 5521856, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 77735} +{"epoch": 0.6640625, "grad_norm": 0.2470037341117859, "learning_rate": 8.194107981329746e-05, "loss": 0.0013115773908793926, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.28, "memory/max_allocated (GiB)": 64.28, "ppl": 1.00131, "step": 170, "tokens/total": 5553272, "tokens/train_per_sec_per_gpu": 8.38, "tokens/trainable": 78179} +{"epoch": 0.66796875, "grad_norm": 0.34732377529144287, "learning_rate": 8.170800355882518e-05, "loss": 0.004490479826927185, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.57, "memory/max_allocated (GiB)": 65.57, "ppl": 1.0045, "step": 171, "tokens/total": 5587776, "tokens/train_per_sec_per_gpu": 8.6, "tokens/trainable": 78660} +{"epoch": 0.671875, "grad_norm": 0.8337541222572327, "learning_rate": 8.147381587530713e-05, "loss": 0.010446088388562202, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.0105, "step": 172, "tokens/total": 5620864, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 79110} +{"epoch": 0.67578125, "grad_norm": 0.9965724945068359, "learning_rate": 8.123852650824877e-05, "loss": 0.006601821165531874, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.00662, "step": 173, "tokens/total": 5653656, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 79576} +{"epoch": 0.6796875, "grad_norm": 0.3336619734764099, "learning_rate": 8.100214524900103e-05, "loss": 0.0018589599058032036, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.09, "memory/max_allocated (GiB)": 63.09, "ppl": 1.00186, "step": 174, "tokens/total": 5682312, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 80001} +{"epoch": 0.68359375, "grad_norm": 1.3976713418960571, "learning_rate": 8.076468193435301e-05, "loss": 0.023773452267050743, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.02406, "step": 175, "tokens/total": 5713520, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 80452} +{"epoch": 0.6875, "grad_norm": 0.8720242381095886, "learning_rate": 8.052614644612253e-05, "loss": 0.00922903697937727, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.73, "memory/max_allocated (GiB)": 64.73, "ppl": 1.00927, "step": 176, "tokens/total": 5745296, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 80886} +{"epoch": 0.69140625, "grad_norm": 1.0958389043807983, "learning_rate": 8.028654871074489e-05, "loss": 0.011508454568684101, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.01157, "step": 177, "tokens/total": 5774336, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 81325} +{"epoch": 0.6953125, "grad_norm": 1.7515385150909424, "learning_rate": 8.004589869885986e-05, "loss": 0.024972595274448395, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.02529, "step": 178, "tokens/total": 5805032, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 81732} +{"epoch": 0.69921875, "grad_norm": 0.8252999186515808, "learning_rate": 7.980420642489674e-05, "loss": 0.008668040856719017, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00871, "step": 179, "tokens/total": 5832616, "tokens/train_per_sec_per_gpu": 10.87, "tokens/trainable": 82171} +{"epoch": 0.703125, "grad_norm": 0.32950806617736816, "learning_rate": 7.95614819466576e-05, "loss": 0.004291071556508541, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.0043, "step": 180, "tokens/total": 5865568, "tokens/train_per_sec_per_gpu": 10.22, "tokens/trainable": 82670} +{"epoch": 0.70703125, "grad_norm": 3.3422811031341553, "learning_rate": 7.931773536489872e-05, "loss": 0.024562731385231018, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.02487, "step": 181, "tokens/total": 5894256, "tokens/train_per_sec_per_gpu": 8.32, "tokens/trainable": 83084} +{"epoch": 0.7109375, "grad_norm": 0.5062534213066101, "learning_rate": 7.907297682291035e-05, "loss": 0.008261105045676231, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.0083, "step": 182, "tokens/total": 5924304, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 83500} +{"epoch": 0.71484375, "grad_norm": 0.48580944538116455, "learning_rate": 7.882721650609442e-05, "loss": 0.005502866581082344, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.01, "memory/max_allocated (GiB)": 64.01, "ppl": 1.00552, "step": 183, "tokens/total": 5955128, "tokens/train_per_sec_per_gpu": 6.99, "tokens/trainable": 83896} +{"epoch": 0.71875, "grad_norm": 0.7360824942588806, "learning_rate": 7.85804646415409e-05, "loss": 0.01286278571933508, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.86, "memory/max_allocated (GiB)": 64.86, "ppl": 1.01295, "step": 184, "tokens/total": 5988056, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 84399} +{"epoch": 0.72265625, "grad_norm": 0.31337612867355347, "learning_rate": 7.833273149760207e-05, "loss": 0.007020048797130585, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00704, "step": 185, "tokens/total": 6019544, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 84873} +{"epoch": 0.7265625, "grad_norm": 0.7192061543464661, "learning_rate": 7.808402738346527e-05, "loss": 0.007881367579102516, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00791, "step": 186, "tokens/total": 6051616, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 85296} +{"epoch": 0.73046875, "grad_norm": 1.6454554796218872, "learning_rate": 7.783436264872382e-05, "loss": 0.006621020846068859, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.54, "memory/max_allocated (GiB)": 64.54, "ppl": 1.00664, "step": 187, "tokens/total": 6084304, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 85741} +{"epoch": 0.734375, "grad_norm": 2.792287826538086, "learning_rate": 7.758374768294647e-05, "loss": 0.005496217869222164, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00551, "step": 188, "tokens/total": 6117896, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 86176} +{"epoch": 0.73828125, "grad_norm": 1.1742188930511475, "learning_rate": 7.733219291524489e-05, "loss": 0.015847835689783096, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.01597, "step": 189, "tokens/total": 6152296, "tokens/train_per_sec_per_gpu": 8.69, "tokens/trainable": 86649} +{"epoch": 0.7421875, "grad_norm": 1.0919580459594727, "learning_rate": 7.707970881383977e-05, "loss": 0.012306725606322289, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.24, "memory/max_allocated (GiB)": 65.24, "ppl": 1.01238, "step": 190, "tokens/total": 6186056, "tokens/train_per_sec_per_gpu": 7.23, "tokens/trainable": 87084} +{"epoch": 0.74609375, "grad_norm": 2.32399582862854, "learning_rate": 7.682630588562518e-05, "loss": 0.015052123926579952, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.01517, "step": 191, "tokens/total": 6219160, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 87499} +{"epoch": 0.75, "grad_norm": 0.5885421633720398, "learning_rate": 7.657199467573129e-05, "loss": 0.0005464035784825683, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00055, "step": 192, "tokens/total": 6254840, "tokens/train_per_sec_per_gpu": 7.27, "tokens/trainable": 87987} +{"epoch": 0.75390625, "grad_norm": 3.4244542121887207, "learning_rate": 7.631678576708561e-05, "loss": 0.014505923725664616, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.01461, "step": 193, "tokens/total": 6289656, "tokens/train_per_sec_per_gpu": 7.32, "tokens/trainable": 88470} +{"epoch": 0.7578125, "grad_norm": 0.5479131937026978, "learning_rate": 7.606068977997255e-05, "loss": 0.0039612408727407455, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00397, "step": 194, "tokens/total": 6321776, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 88923} +{"epoch": 0.76171875, "grad_norm": 0.6855437159538269, "learning_rate": 7.580371737159148e-05, "loss": 0.013426016084849834, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.68, "memory/max_allocated (GiB)": 63.68, "ppl": 1.01352, "step": 195, "tokens/total": 6352824, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 89338} +{"epoch": 0.765625, "grad_norm": 1.0402880907058716, "learning_rate": 7.554587923561324e-05, "loss": 0.021459979936480522, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.02169, "step": 196, "tokens/total": 6388496, "tokens/train_per_sec_per_gpu": 7.74, "tokens/trainable": 89812} +{"epoch": 0.76953125, "grad_norm": 1.226950764656067, "learning_rate": 7.528718610173511e-05, "loss": 0.015128521248698235, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.13, "memory/max_allocated (GiB)": 65.13, "ppl": 1.01524, "step": 197, "tokens/total": 6420600, "tokens/train_per_sec_per_gpu": 9.35, "tokens/trainable": 90252} +{"epoch": 0.7734375, "grad_norm": 1.0312103033065796, "learning_rate": 7.502764873523431e-05, "loss": 0.013678304851055145, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.01377, "step": 198, "tokens/total": 6454616, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 90718} +{"epoch": 0.77734375, "grad_norm": 1.399101972579956, "learning_rate": 7.476727793652011e-05, "loss": 0.008919492363929749, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00896, "step": 199, "tokens/total": 6485680, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 91194} +{"epoch": 0.78125, "grad_norm": 0.1385709047317505, "learning_rate": 7.450608454068415e-05, "loss": 0.0011435108026489615, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00114, "step": 200, "tokens/total": 6518512, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 91649} +{"epoch": 0.78515625, "grad_norm": 1.4206249713897705, "learning_rate": 7.424407941704987e-05, "loss": 0.006891995668411255, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.0, "memory/max_allocated (GiB)": 64.0, "ppl": 1.00692, "step": 201, "tokens/total": 6548728, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 92078} +{"epoch": 0.7890625, "grad_norm": 1.8758479356765747, "learning_rate": 7.398127346871986e-05, "loss": 0.02210191637277603, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.02235, "step": 202, "tokens/total": 6582192, "tokens/train_per_sec_per_gpu": 7.2, "tokens/trainable": 92536} +{"epoch": 0.79296875, "grad_norm": 1.6101349592208862, "learning_rate": 7.371767763212238e-05, "loss": 0.007054885383695364, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.15, "memory/max_allocated (GiB)": 65.15, "ppl": 1.00708, "step": 203, "tokens/total": 6617392, "tokens/train_per_sec_per_gpu": 9.07, "tokens/trainable": 93032} +{"epoch": 0.796875, "grad_norm": 0.3545335829257965, "learning_rate": 7.345330287655617e-05, "loss": 0.005891775246709585, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.03, "memory/max_allocated (GiB)": 64.03, "ppl": 1.00591, "step": 204, "tokens/total": 6648904, "tokens/train_per_sec_per_gpu": 9.67, "tokens/trainable": 93504} +{"epoch": 0.80078125, "grad_norm": 1.2047960758209229, "learning_rate": 7.31881602037339e-05, "loss": 0.02555791288614273, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.17, "memory/max_allocated (GiB)": 64.17, "ppl": 1.02589, "step": 205, "tokens/total": 6679832, "tokens/train_per_sec_per_gpu": 7.41, "tokens/trainable": 93925} +{"epoch": 0.8046875, "grad_norm": 0.5948237180709839, "learning_rate": 7.29222606473245e-05, "loss": 0.009310120716691017, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00935, "step": 206, "tokens/total": 6710784, "tokens/train_per_sec_per_gpu": 9.22, "tokens/trainable": 94365} +{"epoch": 0.80859375, "grad_norm": 0.5116041302680969, "learning_rate": 7.265561527249383e-05, "loss": 0.006107659544795752, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00613, "step": 207, "tokens/total": 6744848, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 94845} +{"epoch": 0.8125, "grad_norm": 0.38595402240753174, "learning_rate": 7.238823517544436e-05, "loss": 0.008222788572311401, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.02, "memory/max_allocated (GiB)": 66.02, "ppl": 1.00826, "step": 208, "tokens/total": 6777600, "tokens/train_per_sec_per_gpu": 8.59, "tokens/trainable": 95285} +{"epoch": 0.81640625, "grad_norm": 0.4453291594982147, "learning_rate": 7.212013148295333e-05, "loss": 0.007336424198001623, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00736, "step": 209, "tokens/total": 6811024, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 95758} +{"epoch": 0.8203125, "grad_norm": 1.1455044746398926, "learning_rate": 7.185131535190975e-05, "loss": 0.008784414269030094, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.15, "memory/max_allocated (GiB)": 66.15, "ppl": 1.00882, "step": 210, "tokens/total": 6845376, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 96192} +{"epoch": 0.82421875, "grad_norm": 2.448866605758667, "learning_rate": 7.158179796885005e-05, "loss": 0.015314456075429916, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.01543, "step": 211, "tokens/total": 6880776, "tokens/train_per_sec_per_gpu": 7.86, "tokens/trainable": 96641} +{"epoch": 0.828125, "grad_norm": 0.6715300679206848, "learning_rate": 7.131159054949273e-05, "loss": 0.014641264453530312, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.01475, "step": 212, "tokens/total": 6916160, "tokens/train_per_sec_per_gpu": 8.92, "tokens/trainable": 97160} +{"epoch": 0.83203125, "grad_norm": 0.3330453932285309, "learning_rate": 7.104070433827139e-05, "loss": 0.0028513309080153704, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.54, "memory/max_allocated (GiB)": 63.54, "ppl": 1.00286, "step": 213, "tokens/total": 6945320, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 97611} +{"epoch": 0.8359375, "grad_norm": 1.3710267543792725, "learning_rate": 7.076915060786705e-05, "loss": 0.00843200832605362, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00847, "step": 214, "tokens/total": 6977856, "tokens/train_per_sec_per_gpu": 9.77, "tokens/trainable": 98111} +{"epoch": 0.83984375, "grad_norm": 0.9161427617073059, "learning_rate": 7.049694065873882e-05, "loss": 0.013698539696633816, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.89, "memory/max_allocated (GiB)": 64.89, "ppl": 1.01379, "step": 215, "tokens/total": 7011096, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 98575} +{"epoch": 0.84375, "grad_norm": 0.6136217713356018, "learning_rate": 7.022408581865382e-05, "loss": 0.009571189060807228, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.93, "memory/max_allocated (GiB)": 63.93, "ppl": 1.00962, "step": 216, "tokens/total": 7041408, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 98991} +{"epoch": 0.84765625, "grad_norm": 0.8703022599220276, "learning_rate": 6.99505974422157e-05, "loss": 0.007725541479885578, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.6, "memory/max_allocated (GiB)": 63.6, "ppl": 1.00776, "step": 217, "tokens/total": 7071560, "tokens/train_per_sec_per_gpu": 11.26, "tokens/trainable": 99454} +{"epoch": 0.8515625, "grad_norm": 0.019704826176166534, "learning_rate": 6.967648691039213e-05, "loss": 0.0002793082967400551, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00028, "step": 218, "tokens/total": 7104696, "tokens/train_per_sec_per_gpu": 8.89, "tokens/trainable": 99925} +{"epoch": 0.85546875, "grad_norm": 0.263949453830719, "learning_rate": 6.940176563004123e-05, "loss": 0.001335658016614616, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00134, "step": 219, "tokens/total": 7138504, "tokens/train_per_sec_per_gpu": 8.86, "tokens/trainable": 100371} +{"epoch": 0.859375, "grad_norm": 0.20432889461517334, "learning_rate": 6.912644503343682e-05, "loss": 0.0031342224683612585, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00314, "step": 220, "tokens/total": 7170992, "tokens/train_per_sec_per_gpu": 9.11, "tokens/trainable": 100855} +{"epoch": 0.86328125, "grad_norm": 0.4851646423339844, "learning_rate": 6.885053657779273e-05, "loss": 0.008861396461725235, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.0089, "step": 221, "tokens/total": 7201176, "tokens/train_per_sec_per_gpu": 6.75, "tokens/trainable": 101253} +{"epoch": 0.8671875, "grad_norm": 0.13856324553489685, "learning_rate": 6.857405174478604e-05, "loss": 0.0016405483474954963, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00164, "step": 222, "tokens/total": 7233528, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 101706} +{"epoch": 0.87109375, "grad_norm": 0.755628228187561, "learning_rate": 6.82970020400792e-05, "loss": 0.01771390810608864, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.01787, "step": 223, "tokens/total": 7267672, "tokens/train_per_sec_per_gpu": 8.18, "tokens/trainable": 102158} +{"epoch": 0.875, "grad_norm": 0.47715631127357483, "learning_rate": 6.801939899284132e-05, "loss": 0.006664336659014225, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 62.86, "memory/max_allocated (GiB)": 62.86, "ppl": 1.00669, "step": 224, "tokens/total": 7295592, "tokens/train_per_sec_per_gpu": 9.15, "tokens/trainable": 102580} +{"epoch": 0.87890625, "grad_norm": 2.629565954208374, "learning_rate": 6.774125415526827e-05, "loss": 0.01934378780424595, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.94, "memory/max_allocated (GiB)": 63.94, "ppl": 1.01953, "step": 225, "tokens/total": 7325816, "tokens/train_per_sec_per_gpu": 9.85, "tokens/trainable": 103063} +{"epoch": 0.8828125, "grad_norm": 1.556557297706604, "learning_rate": 6.746257910210214e-05, "loss": 0.026375774294137955, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.02673, "step": 226, "tokens/total": 7360888, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 103510} +{"epoch": 0.88671875, "grad_norm": 0.24103021621704102, "learning_rate": 6.718338543014937e-05, "loss": 0.004623692017048597, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00463, "step": 227, "tokens/total": 7390416, "tokens/train_per_sec_per_gpu": 10.06, "tokens/trainable": 103980} +{"epoch": 0.890625, "grad_norm": 0.3260801136493683, "learning_rate": 6.69036847577983e-05, "loss": 0.00527721643447876, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.00529, "step": 228, "tokens/total": 7420480, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 104413} +{"epoch": 0.89453125, "grad_norm": 0.2185668796300888, "learning_rate": 6.662348872453553e-05, "loss": 0.002093367278575897, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.0021, "step": 229, "tokens/total": 7448992, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 104845} +{"epoch": 0.8984375, "grad_norm": 0.11743110418319702, "learning_rate": 6.63428089904618e-05, "loss": 0.00198440533131361, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.00199, "step": 230, "tokens/total": 7481744, "tokens/train_per_sec_per_gpu": 7.4, "tokens/trainable": 105289} +{"epoch": 0.90234375, "grad_norm": 1.0220458507537842, "learning_rate": 6.60616572358065e-05, "loss": 0.008907739073038101, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00895, "step": 231, "tokens/total": 7516704, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 105722} +{"epoch": 0.90625, "grad_norm": 1.1773524284362793, "learning_rate": 6.578004516044172e-05, "loss": 0.01043158583343029, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.19, "memory/max_allocated (GiB)": 64.19, "ppl": 1.01049, "step": 232, "tokens/total": 7548592, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 106213} +{"epoch": 0.91015625, "grad_norm": 0.2550044655799866, "learning_rate": 6.549798448339548e-05, "loss": 0.005363100674003363, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00538, "step": 233, "tokens/total": 7579288, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 106643} +{"epoch": 0.9140625, "grad_norm": 0.2716599702835083, "learning_rate": 6.521548694236384e-05, "loss": 0.008243403397500515, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00828, "step": 234, "tokens/total": 7611672, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 107136} +{"epoch": 0.91796875, "grad_norm": 0.4004894196987152, "learning_rate": 6.493256429322259e-05, "loss": 0.007922545075416565, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00795, "step": 235, "tokens/total": 7646288, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 107588} +{"epoch": 0.921875, "grad_norm": 0.736997127532959, "learning_rate": 6.464922830953799e-05, "loss": 0.017940521240234375, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.0181, "step": 236, "tokens/total": 7680136, "tokens/train_per_sec_per_gpu": 7.71, "tokens/trainable": 108087} +{"epoch": 0.92578125, "grad_norm": 0.33834323287010193, "learning_rate": 6.436549078207688e-05, "loss": 0.003915418405085802, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.61, "memory/max_allocated (GiB)": 64.61, "ppl": 1.00392, "step": 237, "tokens/total": 7713072, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 108533} +{"epoch": 0.9296875, "grad_norm": 0.06405025720596313, "learning_rate": 6.408136351831592e-05, "loss": 0.0010499252239242196, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00105, "step": 238, "tokens/total": 7747856, "tokens/train_per_sec_per_gpu": 6.87, "tokens/trainable": 108965} +{"epoch": 0.93359375, "grad_norm": 0.7831677198410034, "learning_rate": 6.379685834195036e-05, "loss": 0.01027410477399826, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.01033, "step": 239, "tokens/total": 7780136, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 109416} +{"epoch": 0.9375, "grad_norm": 0.057659003883600235, "learning_rate": 6.351198709240186e-05, "loss": 0.0008042339468374848, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.78, "memory/max_allocated (GiB)": 65.78, "ppl": 1.0008, "step": 240, "tokens/total": 7816344, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 109864} +{"epoch": 0.94140625, "grad_norm": 0.12719009816646576, "learning_rate": 6.32267616243259e-05, "loss": 0.0017544962465763092, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00176, "step": 241, "tokens/total": 7850408, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 110345} +{"epoch": 0.9453125, "grad_norm": 0.2647528052330017, "learning_rate": 6.294119380711849e-05, "loss": 0.0036112607922405005, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00362, "step": 242, "tokens/total": 7881528, "tokens/train_per_sec_per_gpu": 7.82, "tokens/trainable": 110794} +{"epoch": 0.94921875, "grad_norm": 0.5893134474754333, "learning_rate": 6.265529552442209e-05, "loss": 0.004326450172811747, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00434, "step": 243, "tokens/total": 7915712, "tokens/train_per_sec_per_gpu": 9.92, "tokens/trainable": 111250} +{"epoch": 0.953125, "grad_norm": 0.2858879864215851, "learning_rate": 6.236907867363127e-05, "loss": 0.005881632678210735, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.61, "memory/max_allocated (GiB)": 64.61, "ppl": 1.0059, "step": 244, "tokens/total": 7947408, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 111693} +{"epoch": 0.95703125, "grad_norm": 0.2342933863401413, "learning_rate": 6.208255516539749e-05, "loss": 0.003051615320146084, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.26, "memory/max_allocated (GiB)": 65.26, "ppl": 1.00306, "step": 245, "tokens/total": 7979600, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 112158} +{"epoch": 0.9609375, "grad_norm": 0.8657107949256897, "learning_rate": 6.179573692313344e-05, "loss": 0.01401575654745102, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.01411, "step": 246, "tokens/total": 8010848, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 112601} +{"epoch": 0.96484375, "grad_norm": 0.06202538684010506, "learning_rate": 6.150863588251694e-05, "loss": 0.0007599811069667339, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.7, "memory/max_allocated (GiB)": 63.7, "ppl": 1.00076, "step": 247, "tokens/total": 8040632, "tokens/train_per_sec_per_gpu": 7.37, "tokens/trainable": 113016} +{"epoch": 0.96875, "grad_norm": 0.3237002193927765, "learning_rate": 6.122126399099419e-05, "loss": 0.002450457541272044, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00245, "step": 248, "tokens/total": 8072464, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 113476} +{"epoch": 0.97265625, "grad_norm": 0.15704204142093658, "learning_rate": 6.0933633207282615e-05, "loss": 0.0011625364422798157, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.96, "memory/max_allocated (GiB)": 64.96, "ppl": 1.00116, "step": 249, "tokens/total": 8106568, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 113941} +{"epoch": 0.9765625, "grad_norm": 0.5029094815254211, "learning_rate": 6.064575550087316e-05, "loss": 0.01239662803709507, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.38, "memory/max_allocated (GiB)": 64.38, "ppl": 1.01247, "step": 250, "tokens/total": 8138496, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 114434} +{"epoch": 0.98046875, "grad_norm": 1.1884130239486694, "learning_rate": 6.0357642851532245e-05, "loss": 0.011243644170463085, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.57, "memory/max_allocated (GiB)": 63.57, "ppl": 1.01131, "step": 251, "tokens/total": 8168080, "tokens/train_per_sec_per_gpu": 10.16, "tokens/trainable": 114898} +{"epoch": 0.984375, "grad_norm": 0.17701731622219086, "learning_rate": 6.0069307248803294e-05, "loss": 0.0011678403243422508, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00117, "step": 252, "tokens/total": 8200888, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 115331} +{"epoch": 0.98828125, "grad_norm": 0.02194489911198616, "learning_rate": 5.9780760691507635e-05, "loss": 7.748304051347077e-05, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.92, "memory/max_allocated (GiB)": 63.92, "ppl": 1.00008, "step": 253, "tokens/total": 8230704, "tokens/train_per_sec_per_gpu": 9.65, "tokens/trainable": 115732} +{"epoch": 0.9921875, "grad_norm": 0.30074161291122437, "learning_rate": 5.9492015187245334e-05, "loss": 0.005769784562289715, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00579, "step": 254, "tokens/total": 8266560, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 116221} +{"epoch": 0.99609375, "grad_norm": 0.35156115889549255, "learning_rate": 5.920308275189541e-05, "loss": 0.002171565778553486, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 65.19, "memory/max_allocated (GiB)": 65.19, "ppl": 1.00217, "step": 255, "tokens/total": 8300184, "tokens/train_per_sec_per_gpu": 7.01, "tokens/trainable": 116684} +{"epoch": 1.0, "grad_norm": 0.014786962419748306, "learning_rate": 5.8913975409115874e-05, "loss": 0.00014212506357580423, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00014, "step": 256, "tokens/total": 8333624, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 117126} +{"epoch": 1.00390625, "grad_norm": 0.038007959723472595, "learning_rate": 5.8624705189843395e-05, "loss": 0.000341596023645252, "memory/device_reserved (GiB)": 67.05, "memory/max_active (GiB)": 63.64, "memory/max_allocated (GiB)": 63.64, "ppl": 1.00034, "step": 257, "tokens/total": 8364976, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 117592} +{"epoch": 1.0078125, "grad_norm": 0.2629120349884033, "learning_rate": 5.833528413179249e-05, "loss": 0.0031305463053286076, "memory/device_reserved (GiB)": 66.62, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00314, "step": 258, "tokens/total": 8399568, "tokens/train_per_sec_per_gpu": 6.94, "tokens/trainable": 118030} +{"epoch": 1.01171875, "grad_norm": 0.5129328370094299, "learning_rate": 5.80457242789548e-05, "loss": 0.010106975212693214, "memory/device_reserved (GiB)": 66.62, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.01016, "step": 259, "tokens/total": 8430616, "tokens/train_per_sec_per_gpu": 8.35, "tokens/trainable": 118470} +{"epoch": 1.015625, "grad_norm": 1.2384456396102905, "learning_rate": 5.77560376810977e-05, "loss": 0.005216581746935844, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00523, "step": 260, "tokens/total": 8462696, "tokens/train_per_sec_per_gpu": 9.97, "tokens/trainable": 118935} +{"epoch": 1.01953125, "grad_norm": 0.013783682137727737, "learning_rate": 5.7466236393263005e-05, "loss": 0.0002081938146147877, "memory/device_reserved (GiB)": 66.63, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00021, "step": 261, "tokens/total": 8496152, "tokens/train_per_sec_per_gpu": 7.84, "tokens/trainable": 119381} +{"epoch": 1.0234375, "grad_norm": 0.39940276741981506, "learning_rate": 5.717633247526522e-05, "loss": 0.0122069101780653, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.01228, "step": 262, "tokens/total": 8532912, "tokens/train_per_sec_per_gpu": 7.02, "tokens/trainable": 119849} +{"epoch": 1.02734375, "grad_norm": 0.029877983033657074, "learning_rate": 5.688633799118971e-05, "loss": 0.00019074104784522206, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.00019, "step": 263, "tokens/total": 8565376, "tokens/train_per_sec_per_gpu": 9.78, "tokens/trainable": 120319} +{"epoch": 1.03125, "grad_norm": 0.4763402044773102, "learning_rate": 5.659626500889066e-05, "loss": 0.0052826981991529465, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.0053, "step": 264, "tokens/total": 8597640, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 120804} +{"epoch": 1.03515625, "grad_norm": 0.27012744545936584, "learning_rate": 5.6306125599488905e-05, "loss": 0.0028285998851060867, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.28, "memory/max_allocated (GiB)": 63.28, "ppl": 1.00283, "step": 265, "tokens/total": 8627312, "tokens/train_per_sec_per_gpu": 9.38, "tokens/trainable": 121233} +{"epoch": 1.0390625, "grad_norm": 0.8004207611083984, "learning_rate": 5.601593183686955e-05, "loss": 0.006893157493323088, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.87, "memory/max_allocated (GiB)": 64.87, "ppl": 1.00692, "step": 266, "tokens/total": 8659208, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 121671} +{"epoch": 1.04296875, "grad_norm": 0.2087031453847885, "learning_rate": 5.572569579717961e-05, "loss": 0.0019223808776587248, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.00192, "step": 267, "tokens/total": 8693088, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 122141} +{"epoch": 1.046875, "grad_norm": 0.018979253247380257, "learning_rate": 5.543542955832538e-05, "loss": 0.00016197703371290118, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.58, "memory/max_allocated (GiB)": 65.58, "ppl": 1.00016, "step": 268, "tokens/total": 8725576, "tokens/train_per_sec_per_gpu": 7.38, "tokens/trainable": 122556} +{"epoch": 1.05078125, "grad_norm": 0.2193353921175003, "learning_rate": 5.514514519946986e-05, "loss": 0.004478283692151308, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00449, "step": 269, "tokens/total": 8755192, "tokens/train_per_sec_per_gpu": 10.26, "tokens/trainable": 122988} +{"epoch": 1.0546875, "grad_norm": 0.005143663380295038, "learning_rate": 5.485485480053015e-05, "loss": 6.407861656043679e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00006, "step": 270, "tokens/total": 8787704, "tokens/train_per_sec_per_gpu": 10.46, "tokens/trainable": 123478} +{"epoch": 1.05859375, "grad_norm": 0.1977425068616867, "learning_rate": 5.4564570441674645e-05, "loss": 0.001206608023494482, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.00121, "step": 271, "tokens/total": 8819768, "tokens/train_per_sec_per_gpu": 9.76, "tokens/trainable": 123932} +{"epoch": 1.0625, "grad_norm": 0.023473376408219337, "learning_rate": 5.42743042028204e-05, "loss": 0.00023710154346190393, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.64, "memory/max_allocated (GiB)": 64.64, "ppl": 1.00024, "step": 272, "tokens/total": 8853352, "tokens/train_per_sec_per_gpu": 9.54, "tokens/trainable": 124428} +{"epoch": 1.06640625, "grad_norm": 0.20473751425743103, "learning_rate": 5.3984068163130464e-05, "loss": 0.0015809277538210154, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00158, "step": 273, "tokens/total": 8886840, "tokens/train_per_sec_per_gpu": 7.12, "tokens/trainable": 124866} +{"epoch": 1.0703125, "grad_norm": 0.8941119313240051, "learning_rate": 5.369387440051111e-05, "loss": 0.022341948002576828, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.02259, "step": 274, "tokens/total": 8920808, "tokens/train_per_sec_per_gpu": 7.05, "tokens/trainable": 125324} +{"epoch": 1.07421875, "grad_norm": 0.17432843148708344, "learning_rate": 5.340373499110935e-05, "loss": 0.0008502541459165514, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00085, "step": 275, "tokens/total": 8955880, "tokens/train_per_sec_per_gpu": 9.74, "tokens/trainable": 125813} +{"epoch": 1.078125, "grad_norm": 0.03796864300966263, "learning_rate": 5.3113662008810304e-05, "loss": 0.0001301055308431387, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.77, "memory/max_allocated (GiB)": 65.77, "ppl": 1.00013, "step": 276, "tokens/total": 8989520, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 126252} +{"epoch": 1.08203125, "grad_norm": 0.017220163717865944, "learning_rate": 5.282366752473479e-05, "loss": 0.00017597599071450531, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00018, "step": 277, "tokens/total": 9024680, "tokens/train_per_sec_per_gpu": 7.94, "tokens/trainable": 126696} +{"epoch": 1.0859375, "grad_norm": 0.9217014312744141, "learning_rate": 5.2533763606737005e-05, "loss": 0.0175298023968935, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.13, "memory/max_allocated (GiB)": 64.13, "ppl": 1.01768, "step": 278, "tokens/total": 9056208, "tokens/train_per_sec_per_gpu": 9.5, "tokens/trainable": 127173} +{"epoch": 1.08984375, "grad_norm": 1.2647658586502075, "learning_rate": 5.224396231890232e-05, "loss": 0.011923935264348984, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.012, "step": 279, "tokens/total": 9086376, "tokens/train_per_sec_per_gpu": 9.87, "tokens/trainable": 127637} +{"epoch": 1.09375, "grad_norm": 0.11154526472091675, "learning_rate": 5.195427572104522e-05, "loss": 0.0006839524721726775, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.36, "memory/max_allocated (GiB)": 65.36, "ppl": 1.00068, "step": 280, "tokens/total": 9118944, "tokens/train_per_sec_per_gpu": 8.53, "tokens/trainable": 128112} +{"epoch": 1.09765625, "grad_norm": 0.023117735981941223, "learning_rate": 5.166471586820751e-05, "loss": 0.00032822121283970773, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00033, "step": 281, "tokens/total": 9152920, "tokens/train_per_sec_per_gpu": 6.99, "tokens/trainable": 128584} +{"epoch": 1.1015625, "grad_norm": 0.07019134610891342, "learning_rate": 5.1375294810156615e-05, "loss": 0.00023361005878541619, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00023, "step": 282, "tokens/total": 9185728, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 129051} +{"epoch": 1.10546875, "grad_norm": 0.03181794285774231, "learning_rate": 5.1086024590884144e-05, "loss": 0.0002338740014238283, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.63, "memory/max_allocated (GiB)": 64.63, "ppl": 1.00023, "step": 283, "tokens/total": 9217984, "tokens/train_per_sec_per_gpu": 8.11, "tokens/trainable": 129494} +{"epoch": 1.109375, "grad_norm": 0.03978779539465904, "learning_rate": 5.079691724810461e-05, "loss": 0.0004921511863358319, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00049, "step": 284, "tokens/total": 9253160, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 129983} +{"epoch": 1.11328125, "grad_norm": 0.12303788959980011, "learning_rate": 5.0507984812754684e-05, "loss": 0.0014331991551443934, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00143, "step": 285, "tokens/total": 9286136, "tokens/train_per_sec_per_gpu": 9.03, "tokens/trainable": 130405} +{"epoch": 1.1171875, "grad_norm": 0.3842698037624359, "learning_rate": 5.021923930849237e-05, "loss": 0.0028540808707475662, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00286, "step": 286, "tokens/total": 9318360, "tokens/train_per_sec_per_gpu": 9.27, "tokens/trainable": 130829} +{"epoch": 1.12109375, "grad_norm": 0.14553524553775787, "learning_rate": 4.99306927511967e-05, "loss": 0.00041652534855529666, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.43, "memory/max_allocated (GiB)": 64.43, "ppl": 1.00042, "step": 287, "tokens/total": 9350288, "tokens/train_per_sec_per_gpu": 7.57, "tokens/trainable": 131287} +{"epoch": 1.125, "grad_norm": 0.6404208540916443, "learning_rate": 4.964235714846775e-05, "loss": 0.011286258697509766, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.01135, "step": 288, "tokens/total": 9382104, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 131712} +{"epoch": 1.12890625, "grad_norm": 0.2190258949995041, "learning_rate": 4.9354244499126866e-05, "loss": 0.002149688545614481, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00215, "step": 289, "tokens/total": 9414392, "tokens/train_per_sec_per_gpu": 9.13, "tokens/trainable": 132168} +{"epoch": 1.1328125, "grad_norm": 0.15911446511745453, "learning_rate": 4.90663667927174e-05, "loss": 0.0011855836492031813, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00119, "step": 290, "tokens/total": 9447112, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 132626} +{"epoch": 1.13671875, "grad_norm": 0.8162606358528137, "learning_rate": 4.877873600900581e-05, "loss": 0.012330678291618824, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.01241, "step": 291, "tokens/total": 9483400, "tokens/train_per_sec_per_gpu": 8.73, "tokens/trainable": 133119} +{"epoch": 1.140625, "grad_norm": 0.06497885286808014, "learning_rate": 4.849136411748306e-05, "loss": 0.00020688583026640117, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.48, "memory/max_allocated (GiB)": 64.48, "ppl": 1.00021, "step": 292, "tokens/total": 9513896, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 133550} +{"epoch": 1.14453125, "grad_norm": 0.4080912172794342, "learning_rate": 4.8204263076866574e-05, "loss": 0.003669869853183627, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.59, "memory/max_allocated (GiB)": 65.59, "ppl": 1.00368, "step": 293, "tokens/total": 9550184, "tokens/train_per_sec_per_gpu": 7.76, "tokens/trainable": 133992} +{"epoch": 1.1484375, "grad_norm": 0.3205471336841583, "learning_rate": 4.791744483460251e-05, "loss": 0.0027019924018532038, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00271, "step": 294, "tokens/total": 9583920, "tokens/train_per_sec_per_gpu": 10.04, "tokens/trainable": 134467} +{"epoch": 1.15234375, "grad_norm": 0.2597840428352356, "learning_rate": 4.7630921326368736e-05, "loss": 0.0027324939146637917, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00274, "step": 295, "tokens/total": 9617288, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 134895} +{"epoch": 1.15625, "grad_norm": 0.161873459815979, "learning_rate": 4.7344704475577916e-05, "loss": 0.0014137394027784467, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 62.94, "memory/max_allocated (GiB)": 62.94, "ppl": 1.00141, "step": 296, "tokens/total": 9645600, "tokens/train_per_sec_per_gpu": 9.25, "tokens/trainable": 135321} +{"epoch": 1.16015625, "grad_norm": 0.8714792728424072, "learning_rate": 4.705880619288153e-05, "loss": 0.0015110556269064546, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.76, "memory/max_allocated (GiB)": 64.76, "ppl": 1.00151, "step": 297, "tokens/total": 9679456, "tokens/train_per_sec_per_gpu": 8.24, "tokens/trainable": 135765} +{"epoch": 1.1640625, "grad_norm": 0.010452558286488056, "learning_rate": 4.677323837567412e-05, "loss": 0.00016102523659355938, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.0, "memory/max_allocated (GiB)": 66.0, "ppl": 1.00016, "step": 298, "tokens/total": 9711696, "tokens/train_per_sec_per_gpu": 9.29, "tokens/trainable": 136196} +{"epoch": 1.16796875, "grad_norm": 0.20738962292671204, "learning_rate": 4.6488012907598146e-05, "loss": 0.0018089102813974023, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.92, "memory/max_allocated (GiB)": 65.92, "ppl": 1.00181, "step": 299, "tokens/total": 9745792, "tokens/train_per_sec_per_gpu": 7.51, "tokens/trainable": 136649} +{"epoch": 1.171875, "grad_norm": 1.345889925956726, "learning_rate": 4.620314165804964e-05, "loss": 0.0155818285420537, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.64, "memory/max_allocated (GiB)": 65.64, "ppl": 1.0157, "step": 300, "tokens/total": 9780648, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 137128} +{"epoch": 1.17578125, "grad_norm": 0.3270772695541382, "learning_rate": 4.591863648168407e-05, "loss": 0.0016666966257616878, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00167, "step": 301, "tokens/total": 9813032, "tokens/train_per_sec_per_gpu": 9.12, "tokens/trainable": 137585} +{"epoch": 1.1796875, "grad_norm": 0.33793768286705017, "learning_rate": 4.5634509217923135e-05, "loss": 0.0014369763666763902, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00144, "step": 302, "tokens/total": 9845200, "tokens/train_per_sec_per_gpu": 9.96, "tokens/trainable": 138054} +{"epoch": 1.18359375, "grad_norm": 0.05121174454689026, "learning_rate": 4.535077169046201e-05, "loss": 0.00045123990275897086, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.29, "memory/max_allocated (GiB)": 65.29, "ppl": 1.00045, "step": 303, "tokens/total": 9878616, "tokens/train_per_sec_per_gpu": 9.39, "tokens/trainable": 138559} +{"epoch": 1.1875, "grad_norm": 0.9813287854194641, "learning_rate": 4.506743570677743e-05, "loss": 0.009329746477305889, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.00937, "step": 304, "tokens/total": 9910776, "tokens/train_per_sec_per_gpu": 8.09, "tokens/trainable": 139050} +{"epoch": 1.19140625, "grad_norm": 0.10933700203895569, "learning_rate": 4.478451305763618e-05, "loss": 0.0005298229516483843, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.86, "memory/max_allocated (GiB)": 65.86, "ppl": 1.00053, "step": 305, "tokens/total": 9945480, "tokens/train_per_sec_per_gpu": 8.51, "tokens/trainable": 139525} +{"epoch": 1.1953125, "grad_norm": 0.10068654268980026, "learning_rate": 4.450201551660454e-05, "loss": 0.0008015864295884967, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.0008, "step": 306, "tokens/total": 9975992, "tokens/train_per_sec_per_gpu": 8.52, "tokens/trainable": 139969} +{"epoch": 1.19921875, "grad_norm": 0.12145973742008209, "learning_rate": 4.4219954839558276e-05, "loss": 0.0007348000071942806, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.69, "memory/max_allocated (GiB)": 63.69, "ppl": 1.00074, "step": 307, "tokens/total": 10006440, "tokens/train_per_sec_per_gpu": 9.83, "tokens/trainable": 140443} +{"epoch": 1.203125, "grad_norm": 0.02908017672598362, "learning_rate": 4.393834276419352e-05, "loss": 0.0002474010398145765, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00025, "step": 308, "tokens/total": 10042928, "tokens/train_per_sec_per_gpu": 7.85, "tokens/trainable": 140926} +{"epoch": 1.20703125, "grad_norm": 0.4836011528968811, "learning_rate": 4.36571910095382e-05, "loss": 0.001598094473592937, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.79, "memory/max_allocated (GiB)": 65.79, "ppl": 1.0016, "step": 309, "tokens/total": 10075184, "tokens/train_per_sec_per_gpu": 8.02, "tokens/trainable": 141344} +{"epoch": 1.2109375, "grad_norm": 0.6539953351020813, "learning_rate": 4.337651127546448e-05, "loss": 0.021059243008494377, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.02128, "step": 310, "tokens/total": 10106656, "tokens/train_per_sec_per_gpu": 8.19, "tokens/trainable": 141753} +{"epoch": 1.21484375, "grad_norm": 0.7637205123901367, "learning_rate": 4.3096315242201736e-05, "loss": 0.009624832309782505, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.82, "memory/max_allocated (GiB)": 63.82, "ppl": 1.00967, "step": 311, "tokens/total": 10138536, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 142219} +{"epoch": 1.21875, "grad_norm": 0.3519359230995178, "learning_rate": 4.2816614569850635e-05, "loss": 0.010983488522469997, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.74, "memory/max_allocated (GiB)": 63.74, "ppl": 1.01104, "step": 312, "tokens/total": 10165768, "tokens/train_per_sec_per_gpu": 8.68, "tokens/trainable": 142617} +{"epoch": 1.22265625, "grad_norm": 0.1878662258386612, "learning_rate": 4.2537420897897864e-05, "loss": 0.0008387328125536442, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00084, "step": 313, "tokens/total": 10201624, "tokens/train_per_sec_per_gpu": 7.19, "tokens/trainable": 143103} +{"epoch": 1.2265625, "grad_norm": 0.07791382074356079, "learning_rate": 4.225874584473174e-05, "loss": 0.0007857225136831403, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.14, "memory/max_allocated (GiB)": 64.14, "ppl": 1.00079, "step": 314, "tokens/total": 10233304, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 143535} +{"epoch": 1.23046875, "grad_norm": 1.2634024620056152, "learning_rate": 4.19806010071587e-05, "loss": 0.0025616565253585577, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.00256, "step": 315, "tokens/total": 10267904, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 144009} +{"epoch": 1.234375, "grad_norm": 0.44908037781715393, "learning_rate": 4.170299795992081e-05, "loss": 0.0030072820372879505, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00301, "step": 316, "tokens/total": 10301280, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 144460} +{"epoch": 1.23828125, "grad_norm": 0.1898353546857834, "learning_rate": 4.142594825521398e-05, "loss": 0.0030528514180332422, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.6, "memory/max_allocated (GiB)": 65.6, "ppl": 1.00306, "step": 317, "tokens/total": 10337072, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 144936} +{"epoch": 1.2421875, "grad_norm": 0.04456224665045738, "learning_rate": 4.114946342220728e-05, "loss": 0.000295601028483361, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.69, "memory/max_allocated (GiB)": 64.69, "ppl": 1.0003, "step": 318, "tokens/total": 10368688, "tokens/train_per_sec_per_gpu": 8.27, "tokens/trainable": 145375} +{"epoch": 1.24609375, "grad_norm": 0.07726289331912994, "learning_rate": 4.087355496656321e-05, "loss": 0.0012155004078522325, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00122, "step": 319, "tokens/total": 10400752, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 145849} +{"epoch": 1.25, "grad_norm": 0.09257698804140091, "learning_rate": 4.05982343699588e-05, "loss": 0.0007503809756599367, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00075, "step": 320, "tokens/total": 10432112, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 146318} +{"epoch": 1.25390625, "grad_norm": 0.1240072250366211, "learning_rate": 4.0323513089607876e-05, "loss": 0.0006554881692864001, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.85, "memory/max_allocated (GiB)": 65.85, "ppl": 1.00066, "step": 321, "tokens/total": 10465472, "tokens/train_per_sec_per_gpu": 9.59, "tokens/trainable": 146821} +{"epoch": 1.2578125, "grad_norm": 0.2713838517665863, "learning_rate": 4.004940255778431e-05, "loss": 0.0012036762200295925, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.42, "memory/max_allocated (GiB)": 63.42, "ppl": 1.0012, "step": 322, "tokens/total": 10496336, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 147285} +{"epoch": 1.26171875, "grad_norm": 0.02749757282435894, "learning_rate": 3.977591418134619e-05, "loss": 0.00036613710108213127, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.45, "memory/max_allocated (GiB)": 63.45, "ppl": 1.00037, "step": 323, "tokens/total": 10525864, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 147729} +{"epoch": 1.265625, "grad_norm": 0.4836643934249878, "learning_rate": 3.95030593412612e-05, "loss": 0.006196931935846806, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.55, "memory/max_allocated (GiB)": 65.55, "ppl": 1.00622, "step": 324, "tokens/total": 10559680, "tokens/train_per_sec_per_gpu": 8.34, "tokens/trainable": 148169} +{"epoch": 1.26953125, "grad_norm": 0.28477492928504944, "learning_rate": 3.923084939213296e-05, "loss": 0.0020062243565917015, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00201, "step": 325, "tokens/total": 10595400, "tokens/train_per_sec_per_gpu": 7.26, "tokens/trainable": 148630} +{"epoch": 1.2734375, "grad_norm": 0.3107934594154358, "learning_rate": 3.895929566172861e-05, "loss": 0.006399292964488268, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.51, "memory/max_allocated (GiB)": 64.51, "ppl": 1.00642, "step": 326, "tokens/total": 10626680, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 149063} +{"epoch": 1.27734375, "grad_norm": 0.08987655490636826, "learning_rate": 3.868840945050728e-05, "loss": 0.0003980436595156789, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.0004, "step": 327, "tokens/total": 10661784, "tokens/train_per_sec_per_gpu": 7.77, "tokens/trainable": 149549} +{"epoch": 1.28125, "grad_norm": 0.2838176488876343, "learning_rate": 3.841820203114995e-05, "loss": 0.003924066200852394, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.31, "memory/max_allocated (GiB)": 65.31, "ppl": 1.00393, "step": 328, "tokens/total": 10696184, "tokens/train_per_sec_per_gpu": 7.97, "tokens/trainable": 150015} +{"epoch": 1.28515625, "grad_norm": 0.20651109516620636, "learning_rate": 3.814868464809027e-05, "loss": 0.001153398654423654, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00115, "step": 329, "tokens/total": 10729376, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 150514} +{"epoch": 1.2890625, "grad_norm": 0.0070175593718886375, "learning_rate": 3.787986851704667e-05, "loss": 7.810996612533927e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.52, "memory/max_allocated (GiB)": 65.52, "ppl": 1.00008, "step": 330, "tokens/total": 10764824, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 151021} +{"epoch": 1.29296875, "grad_norm": 0.6645070314407349, "learning_rate": 3.7611764824555654e-05, "loss": 0.010876193642616272, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.01094, "step": 331, "tokens/total": 10797072, "tokens/train_per_sec_per_gpu": 9.88, "tokens/trainable": 151486} +{"epoch": 1.296875, "grad_norm": 0.5440879464149475, "learning_rate": 3.734438472750619e-05, "loss": 0.0106546925380826, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.01071, "step": 332, "tokens/total": 10829376, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 151956} +{"epoch": 1.30078125, "grad_norm": 0.010818447917699814, "learning_rate": 3.707773935267552e-05, "loss": 0.00019067640823777765, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.34, "memory/max_allocated (GiB)": 65.34, "ppl": 1.00019, "step": 333, "tokens/total": 10863512, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 152442} +{"epoch": 1.3046875, "grad_norm": 0.12446974962949753, "learning_rate": 3.68118397962661e-05, "loss": 0.0006918520084582269, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.65, "memory/max_allocated (GiB)": 63.65, "ppl": 1.00069, "step": 334, "tokens/total": 10894432, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 152890} +{"epoch": 1.30859375, "grad_norm": 0.24180521070957184, "learning_rate": 3.654669712344384e-05, "loss": 0.0030620540492236614, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.34, "memory/max_allocated (GiB)": 63.34, "ppl": 1.00307, "step": 335, "tokens/total": 10924848, "tokens/train_per_sec_per_gpu": 8.95, "tokens/trainable": 153368} +{"epoch": 1.3125, "grad_norm": 0.0868603065609932, "learning_rate": 3.628232236787763e-05, "loss": 0.0011242118198424578, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.00112, "step": 336, "tokens/total": 10959024, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 153809} +{"epoch": 1.31640625, "grad_norm": 0.6321369409561157, "learning_rate": 3.6018726531280144e-05, "loss": 0.00475310068577528, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.09, "memory/max_allocated (GiB)": 64.09, "ppl": 1.00476, "step": 337, "tokens/total": 10991088, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 154258} +{"epoch": 1.3203125, "grad_norm": 0.22731904685497284, "learning_rate": 3.575592058295017e-05, "loss": 0.002498295623809099, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.5, "memory/max_allocated (GiB)": 64.5, "ppl": 1.0025, "step": 338, "tokens/total": 11023904, "tokens/train_per_sec_per_gpu": 7.83, "tokens/trainable": 154704} +{"epoch": 1.32421875, "grad_norm": 0.0769808366894722, "learning_rate": 3.549391545931585e-05, "loss": 0.0005408978904597461, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00054, "step": 339, "tokens/total": 11057680, "tokens/train_per_sec_per_gpu": 7.25, "tokens/trainable": 155157} +{"epoch": 1.328125, "grad_norm": 0.11305997520685196, "learning_rate": 3.5232722063479914e-05, "loss": 0.000980703509412706, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00098, "step": 340, "tokens/total": 11089800, "tokens/train_per_sec_per_gpu": 7.45, "tokens/trainable": 155582} +{"epoch": 1.33203125, "grad_norm": 0.07387830317020416, "learning_rate": 3.49723512647657e-05, "loss": 0.0004902435466647148, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00049, "step": 341, "tokens/total": 11117960, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 156047} +{"epoch": 1.3359375, "grad_norm": 0.295072466135025, "learning_rate": 3.471281389826491e-05, "loss": 0.005936713889241219, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00595, "step": 342, "tokens/total": 11151888, "tokens/train_per_sec_per_gpu": 8.31, "tokens/trainable": 156500} +{"epoch": 1.33984375, "grad_norm": 0.0328228697180748, "learning_rate": 3.4454120764386764e-05, "loss": 0.00019401026656851172, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.22, "memory/max_allocated (GiB)": 65.22, "ppl": 1.00019, "step": 343, "tokens/total": 11184568, "tokens/train_per_sec_per_gpu": 8.01, "tokens/trainable": 156931} +{"epoch": 1.34375, "grad_norm": 0.38090598583221436, "learning_rate": 3.4196282628408526e-05, "loss": 0.000993996625766158, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.75, "memory/max_allocated (GiB)": 63.75, "ppl": 1.00099, "step": 344, "tokens/total": 11216400, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 157385} +{"epoch": 1.34765625, "grad_norm": 0.01903524249792099, "learning_rate": 3.3939310220027456e-05, "loss": 0.0001412740966770798, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.12, "memory/max_allocated (GiB)": 66.12, "ppl": 1.00014, "step": 345, "tokens/total": 11250768, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 157817} +{"epoch": 1.3515625, "grad_norm": 0.03523644432425499, "learning_rate": 3.3683214232914404e-05, "loss": 0.0004293542879167944, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00043, "step": 346, "tokens/total": 11281808, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 158273} +{"epoch": 1.35546875, "grad_norm": 0.03901255130767822, "learning_rate": 3.342800532426873e-05, "loss": 0.0004065888933837414, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.23, "memory/max_allocated (GiB)": 64.23, "ppl": 1.00041, "step": 347, "tokens/total": 11313968, "tokens/train_per_sec_per_gpu": 9.8, "tokens/trainable": 158728} +{"epoch": 1.359375, "grad_norm": 0.020742859691381454, "learning_rate": 3.317369411437484e-05, "loss": 0.00023270913516171277, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.6, "memory/max_allocated (GiB)": 64.6, "ppl": 1.00023, "step": 348, "tokens/total": 11347288, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 159168} +{"epoch": 1.36328125, "grad_norm": 0.015057416632771492, "learning_rate": 3.292029118616024e-05, "loss": 0.0001616387307876721, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.99, "memory/max_allocated (GiB)": 65.99, "ppl": 1.00016, "step": 349, "tokens/total": 11377128, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 159592} +{"epoch": 1.3671875, "grad_norm": 0.08861009031534195, "learning_rate": 3.266780708475511e-05, "loss": 0.00037193336174823344, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.41, "memory/max_allocated (GiB)": 64.41, "ppl": 1.00037, "step": 350, "tokens/total": 11409488, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 160093} +{"epoch": 1.37109375, "grad_norm": 0.13333335518836975, "learning_rate": 3.241625231705354e-05, "loss": 0.0007645260775461793, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00076, "step": 351, "tokens/total": 11442040, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 160534} +{"epoch": 1.375, "grad_norm": 0.6730687022209167, "learning_rate": 3.216563735127618e-05, "loss": 0.010844479314982891, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.0109, "step": 352, "tokens/total": 11478312, "tokens/train_per_sec_per_gpu": 7.64, "tokens/trainable": 161009} +{"epoch": 1.37890625, "grad_norm": 0.07716026902198792, "learning_rate": 3.191597261653475e-05, "loss": 0.0007416786975227296, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.93, "memory/max_allocated (GiB)": 63.93, "ppl": 1.00074, "step": 353, "tokens/total": 11509008, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 161450} +{"epoch": 1.3828125, "grad_norm": 0.11327984184026718, "learning_rate": 3.166726850239794e-05, "loss": 0.0007717227563261986, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00077, "step": 354, "tokens/total": 11541264, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 161900} +{"epoch": 1.38671875, "grad_norm": 0.0019182817777618766, "learning_rate": 3.141953535845912e-05, "loss": 1.8056854969472624e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00002, "step": 355, "tokens/total": 11573072, "tokens/train_per_sec_per_gpu": 10.02, "tokens/trainable": 162358} +{"epoch": 1.390625, "grad_norm": 1.2182121276855469, "learning_rate": 3.11727834939056e-05, "loss": 0.012073135934770107, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.08, "memory/max_allocated (GiB)": 64.08, "ppl": 1.01215, "step": 356, "tokens/total": 11605224, "tokens/train_per_sec_per_gpu": 8.0, "tokens/trainable": 162845} +{"epoch": 1.39453125, "grad_norm": 0.08079471439123154, "learning_rate": 3.092702317708967e-05, "loss": 0.0008047674782574177, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00081, "step": 357, "tokens/total": 11637512, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 163299} +{"epoch": 1.3984375, "grad_norm": 0.06098448485136032, "learning_rate": 3.0682264635101276e-05, "loss": 0.0003801141574513167, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.4, "memory/max_allocated (GiB)": 64.4, "ppl": 1.00038, "step": 358, "tokens/total": 11671280, "tokens/train_per_sec_per_gpu": 8.22, "tokens/trainable": 163774} +{"epoch": 1.40234375, "grad_norm": 0.8099466562271118, "learning_rate": 3.0438518053342407e-05, "loss": 0.004541903734207153, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00455, "step": 359, "tokens/total": 11702008, "tokens/train_per_sec_per_gpu": 7.3, "tokens/trainable": 164199} +{"epoch": 1.40625, "grad_norm": 0.5863710045814514, "learning_rate": 3.0195793575103266e-05, "loss": 0.003150820266455412, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.43, "memory/max_allocated (GiB)": 63.43, "ppl": 1.00316, "step": 360, "tokens/total": 11732032, "tokens/train_per_sec_per_gpu": 9.94, "tokens/trainable": 164667} +{"epoch": 1.41015625, "grad_norm": 0.018501611426472664, "learning_rate": 2.9954101301140146e-05, "loss": 0.00024896342074498534, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00025, "step": 361, "tokens/total": 11767472, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.6041479706764221, "learning_rate": 2.9713451289255123e-05, "loss": 0.0030849550385028124, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.56, "memory/max_allocated (GiB)": 64.56, "ppl": 1.00309, "step": 362, "tokens/total": 11798432, "tokens/train_per_sec_per_gpu": 10.39, "tokens/trainable": 165575} +{"epoch": 1.41796875, "grad_norm": 0.9936931729316711, "learning_rate": 2.9473853553877484e-05, "loss": 0.00521257845684886, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.11, "memory/max_allocated (GiB)": 65.11, "ppl": 1.00523, "step": 363, "tokens/total": 11833144, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 166060} +{"epoch": 1.421875, "grad_norm": 0.42553216218948364, "learning_rate": 2.9235318065647e-05, "loss": 0.01341431774199009, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.92, "memory/max_allocated (GiB)": 64.92, "ppl": 1.0135, "step": 364, "tokens/total": 11865856, "tokens/train_per_sec_per_gpu": 7.92, "tokens/trainable": 166478} +{"epoch": 1.42578125, "grad_norm": 0.10351817309856415, "learning_rate": 2.8997854750998964e-05, "loss": 0.00025923154316842556, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.89, "memory/max_allocated (GiB)": 65.89, "ppl": 1.00026, "step": 365, "tokens/total": 11900400, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 166941} +{"epoch": 1.4296875, "grad_norm": 0.031225256621837616, "learning_rate": 2.8761473491751258e-05, "loss": 7.693937368458137e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.49, "memory/max_allocated (GiB)": 63.49, "ppl": 1.00008, "step": 366, "tokens/total": 11931368, "tokens/train_per_sec_per_gpu": 9.66, "tokens/trainable": 167381} +{"epoch": 1.43359375, "grad_norm": 0.5792052745819092, "learning_rate": 2.8526184124692883e-05, "loss": 0.005589433014392853, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.01, "memory/max_allocated (GiB)": 65.01, "ppl": 1.00561, "step": 367, "tokens/total": 11963320, "tokens/train_per_sec_per_gpu": 8.17, "tokens/trainable": 167830} +{"epoch": 1.4375, "grad_norm": 0.08958517760038376, "learning_rate": 2.829199644117484e-05, "loss": 0.0005538593395613134, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.74, "memory/max_allocated (GiB)": 65.74, "ppl": 1.00055, "step": 368, "tokens/total": 11997536, "tokens/train_per_sec_per_gpu": 7.54, "tokens/trainable": 168266} +{"epoch": 1.44140625, "grad_norm": 0.005006122402846813, "learning_rate": 2.8058920186702553e-05, "loss": 7.736667612334713e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00008, "step": 369, "tokens/total": 12029528, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 168716} +{"epoch": 1.4453125, "grad_norm": 0.014643846079707146, "learning_rate": 2.782696506053033e-05, "loss": 0.00015074267867021263, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00015, "step": 370, "tokens/total": 12062504, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 169176} +{"epoch": 1.44921875, "grad_norm": 0.009969312697649002, "learning_rate": 2.7596140715257824e-05, "loss": 0.0001660179696045816, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.78, "memory/max_allocated (GiB)": 65.78, "ppl": 1.00017, "step": 371, "tokens/total": 12097376, "tokens/train_per_sec_per_gpu": 7.48, "tokens/trainable": 169591} +{"epoch": 1.453125, "grad_norm": 0.02798745036125183, "learning_rate": 2.7366456756428184e-05, "loss": 0.0003464683541096747, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.59, "memory/max_allocated (GiB)": 64.59, "ppl": 1.00035, "step": 372, "tokens/total": 12131920, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 170041} +{"epoch": 1.45703125, "grad_norm": 0.5675070285797119, "learning_rate": 2.7137922742128486e-05, "loss": 0.0027816949877887964, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.67, "memory/max_allocated (GiB)": 64.67, "ppl": 1.00279, "step": 373, "tokens/total": 12164528, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 170516} +{"epoch": 1.4609375, "grad_norm": 0.06181880459189415, "learning_rate": 2.691054818259188e-05, "loss": 0.00031025143107399344, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.08, "memory/max_allocated (GiB)": 65.08, "ppl": 1.00031, "step": 374, "tokens/total": 12197376, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 170952} +{"epoch": 1.46484375, "grad_norm": 0.320768266916275, "learning_rate": 2.6684342539801933e-05, "loss": 0.0021169735118746758, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00212, "step": 375, "tokens/total": 12233104, "tokens/train_per_sec_per_gpu": 7.14, "tokens/trainable": 171424} +{"epoch": 1.46875, "grad_norm": 0.4131838381290436, "learning_rate": 2.645931522709877e-05, "loss": 0.0028826636262238026, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.69, "memory/max_allocated (GiB)": 65.69, "ppl": 1.00289, "step": 376, "tokens/total": 12269224, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 171884} +{"epoch": 1.47265625, "grad_norm": 0.03785351663827896, "learning_rate": 2.6235475608787365e-05, "loss": 0.00015159953909460455, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.12, "memory/max_allocated (GiB)": 65.12, "ppl": 1.00015, "step": 377, "tokens/total": 12302952, "tokens/train_per_sec_per_gpu": 9.4, "tokens/trainable": 172404} +{"epoch": 1.4765625, "grad_norm": 0.10668928176164627, "learning_rate": 2.6012832999747916e-05, "loss": 0.0012814282672479749, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.07, "memory/max_allocated (GiB)": 66.07, "ppl": 1.00128, "step": 378, "tokens/total": 12338424, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 172871} +{"epoch": 1.48046875, "grad_norm": 0.042822856456041336, "learning_rate": 2.579139666504821e-05, "loss": 0.00029525114223361015, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.29, "memory/max_allocated (GiB)": 64.29, "ppl": 1.0003, "step": 379, "tokens/total": 12369144, "tokens/train_per_sec_per_gpu": 7.63, "tokens/trainable": 173329} +{"epoch": 1.484375, "grad_norm": 0.02380918711423874, "learning_rate": 2.557117581955798e-05, "loss": 0.0003296207869425416, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00033, "step": 380, "tokens/total": 12403352, "tokens/train_per_sec_per_gpu": 7.79, "tokens/trainable": 173827} +{"epoch": 1.48828125, "grad_norm": 0.012655185535550117, "learning_rate": 2.5352179627565532e-05, "loss": 0.00013097832561470568, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00013, "step": 381, "tokens/total": 12441056, "tokens/train_per_sec_per_gpu": 7.93, "tokens/trainable": 174287} +{"epoch": 1.4921875, "grad_norm": 6.1950860023498535, "learning_rate": 2.5134417202396277e-05, "loss": 0.007711904589086771, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.16, "memory/max_allocated (GiB)": 65.16, "ppl": 1.00774, "step": 382, "tokens/total": 12474080, "tokens/train_per_sec_per_gpu": 7.88, "tokens/trainable": 174731} +{"epoch": 1.49609375, "grad_norm": 0.011765087023377419, "learning_rate": 2.491789760603361e-05, "loss": 0.0001265132159460336, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00013, "step": 383, "tokens/total": 12508992, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 175183} +{"epoch": 1.5, "grad_norm": 0.016618305817246437, "learning_rate": 2.4702629848741764e-05, "loss": 0.00016725876776035875, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.0, "memory/max_allocated (GiB)": 65.0, "ppl": 1.00017, "step": 384, "tokens/total": 12540896, "tokens/train_per_sec_per_gpu": 9.26, "tokens/trainable": 175655} +{"epoch": 1.50390625, "grad_norm": 0.6027182340621948, "learning_rate": 2.4488622888690785e-05, "loss": 0.007934200577437878, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.74, "memory/max_allocated (GiB)": 64.74, "ppl": 1.00797, "step": 385, "tokens/total": 12568416, "tokens/train_per_sec_per_gpu": 9.34, "tokens/trainable": 176085} +{"epoch": 1.5078125, "grad_norm": 0.009245755150914192, "learning_rate": 2.427588563158384e-05, "loss": 7.528211426688358e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.68, "memory/max_allocated (GiB)": 64.68, "ppl": 1.00008, "step": 386, "tokens/total": 12599792, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 176549} +{"epoch": 1.51171875, "grad_norm": 0.09642230719327927, "learning_rate": 2.406442693028651e-05, "loss": 0.0005116576794534922, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.00051, "step": 387, "tokens/total": 12631224, "tokens/train_per_sec_per_gpu": 9.52, "tokens/trainable": 176995} +{"epoch": 1.515625, "grad_norm": 0.011744176037609577, "learning_rate": 2.3854255584458547e-05, "loss": 0.00010907409887295216, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.88, "memory/max_allocated (GiB)": 65.88, "ppl": 1.00011, "step": 388, "tokens/total": 12667240, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 177468} +{"epoch": 1.51953125, "grad_norm": 0.24873362481594086, "learning_rate": 2.3645380340187508e-05, "loss": 0.0032493562903255224, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.54, "memory/max_allocated (GiB)": 65.54, "ppl": 1.00325, "step": 389, "tokens/total": 12700488, "tokens/train_per_sec_per_gpu": 9.41, "tokens/trainable": 177961} +{"epoch": 1.5234375, "grad_norm": 0.04651312902569771, "learning_rate": 2.3437809889624914e-05, "loss": 0.00019797016284428537, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.0002, "step": 390, "tokens/total": 12734528, "tokens/train_per_sec_per_gpu": 6.91, "tokens/trainable": 178417} +{"epoch": 1.52734375, "grad_norm": 0.26920145750045776, "learning_rate": 2.3231552870624487e-05, "loss": 0.0020582498982548714, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00206, "step": 391, "tokens/total": 12767152, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 178862} +{"epoch": 1.53125, "grad_norm": 0.00704180495813489, "learning_rate": 2.3026617866382657e-05, "loss": 0.00011391100997570902, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.31, "memory/max_allocated (GiB)": 64.31, "ppl": 1.00011, "step": 392, "tokens/total": 12798168, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 179320} +{"epoch": 1.53515625, "grad_norm": 0.030999816954135895, "learning_rate": 2.2823013405081507e-05, "loss": 0.000410491949878633, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00041, "step": 393, "tokens/total": 12828896, "tokens/train_per_sec_per_gpu": 9.82, "tokens/trainable": 179758} +{"epoch": 1.5390625, "grad_norm": 0.7445809841156006, "learning_rate": 2.2620747959533722e-05, "loss": 0.005722487345337868, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.71, "memory/max_allocated (GiB)": 63.71, "ppl": 1.00574, "step": 394, "tokens/total": 12860024, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 180211} +{"epoch": 1.54296875, "grad_norm": 0.20269298553466797, "learning_rate": 2.2419829946830123e-05, "loss": 0.0014115928206592798, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00141, "step": 395, "tokens/total": 12891472, "tokens/train_per_sec_per_gpu": 7.68, "tokens/trainable": 180637} +{"epoch": 1.546875, "grad_norm": 0.043014928698539734, "learning_rate": 2.2220267727989325e-05, "loss": 0.00014642247697338462, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.93, "memory/max_allocated (GiB)": 65.93, "ppl": 1.00015, "step": 396, "tokens/total": 12925648, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 181142} +{"epoch": 1.55078125, "grad_norm": 0.03388587012887001, "learning_rate": 2.202206960760984e-05, "loss": 0.0005354695022106171, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.77, "memory/max_allocated (GiB)": 64.77, "ppl": 1.00054, "step": 397, "tokens/total": 12959200, "tokens/train_per_sec_per_gpu": 7.72, "tokens/trainable": 181597} +{"epoch": 1.5546875, "grad_norm": 0.04147177189588547, "learning_rate": 2.182524383352446e-05, "loss": 0.00021648462279699743, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00022, "step": 398, "tokens/total": 12989624, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 182030} +{"epoch": 1.55859375, "grad_norm": 0.03449082374572754, "learning_rate": 2.1629798596457056e-05, "loss": 0.0003568175307009369, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 62.97, "memory/max_allocated (GiB)": 62.97, "ppl": 1.00036, "step": 399, "tokens/total": 13019040, "tokens/train_per_sec_per_gpu": 10.25, "tokens/trainable": 182496} +{"epoch": 1.5625, "grad_norm": 0.03290824219584465, "learning_rate": 2.1435742029681725e-05, "loss": 0.00019418373994994909, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.1, "memory/max_allocated (GiB)": 64.1, "ppl": 1.00019, "step": 400, "tokens/total": 13051808, "tokens/train_per_sec_per_gpu": 8.54, "tokens/trainable": 182963} +{"epoch": 1.56640625, "grad_norm": 0.2781409025192261, "learning_rate": 2.124308220868431e-05, "loss": 0.0015235631726682186, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.23, "memory/max_allocated (GiB)": 65.23, "ppl": 1.00152, "step": 401, "tokens/total": 13085520, "tokens/train_per_sec_per_gpu": 8.26, "tokens/trainable": 183416} +{"epoch": 1.5703125, "grad_norm": 0.006440761964768171, "learning_rate": 2.105182715082638e-05, "loss": 8.503657591063529e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00009, "step": 402, "tokens/total": 13116224, "tokens/train_per_sec_per_gpu": 8.47, "tokens/trainable": 183857} +{"epoch": 1.57421875, "grad_norm": 0.031164519488811493, "learning_rate": 2.0861984815011552e-05, "loss": 0.00018459033162798733, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.83, "memory/max_allocated (GiB)": 64.83, "ppl": 1.00018, "step": 403, "tokens/total": 13149984, "tokens/train_per_sec_per_gpu": 7.52, "tokens/trainable": 184296} +{"epoch": 1.578125, "grad_norm": 0.08721276372671127, "learning_rate": 2.0673563101354323e-05, "loss": 0.00044958043145015836, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00045, "step": 404, "tokens/total": 13185488, "tokens/train_per_sec_per_gpu": 7.8, "tokens/trainable": 184719} +{"epoch": 1.58203125, "grad_norm": 0.007792145013809204, "learning_rate": 2.0486569850851317e-05, "loss": 8.329434058396146e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.26, "memory/max_allocated (GiB)": 64.26, "ppl": 1.00008, "step": 405, "tokens/total": 13216320, "tokens/train_per_sec_per_gpu": 8.97, "tokens/trainable": 185172} +{"epoch": 1.5859375, "grad_norm": 0.4191281199455261, "learning_rate": 2.0301012845054956e-05, "loss": 0.008369292132556438, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.0084, "step": 406, "tokens/total": 13248136, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 185646} +{"epoch": 1.58984375, "grad_norm": 0.32980719208717346, "learning_rate": 2.011689980574966e-05, "loss": 0.001617034780792892, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.42, "memory/max_allocated (GiB)": 64.42, "ppl": 1.00162, "step": 407, "tokens/total": 13280640, "tokens/train_per_sec_per_gpu": 8.04, "tokens/trainable": 186129} +{"epoch": 1.59375, "grad_norm": 0.03538071736693382, "learning_rate": 1.993423839463052e-05, "loss": 0.00030863008578307927, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.2, "memory/max_allocated (GiB)": 65.2, "ppl": 1.00031, "step": 408, "tokens/total": 13314160, "tokens/train_per_sec_per_gpu": 8.21, "tokens/trainable": 186610} +{"epoch": 1.59765625, "grad_norm": 0.04287126660346985, "learning_rate": 1.975303621298445e-05, "loss": 0.00029071501921862364, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.51, "memory/max_allocated (GiB)": 63.51, "ppl": 1.00029, "step": 409, "tokens/total": 13345024, "tokens/train_per_sec_per_gpu": 8.99, "tokens/trainable": 187066} +{"epoch": 1.6015625, "grad_norm": 0.008144103921949863, "learning_rate": 1.957330080137385e-05, "loss": 0.00013642600970342755, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00014, "step": 410, "tokens/total": 13377248, "tokens/train_per_sec_per_gpu": 9.04, "tokens/trainable": 187567} +{"epoch": 1.60546875, "grad_norm": 0.021344207227230072, "learning_rate": 1.9395039639322864e-05, "loss": 0.00022608089784625918, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00023, "step": 411, "tokens/total": 13410992, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 188056} +{"epoch": 1.609375, "grad_norm": 0.024294538423419, "learning_rate": 1.9218260145006073e-05, "loss": 0.00024433867656625807, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.71, "memory/max_allocated (GiB)": 64.71, "ppl": 1.00024, "step": 412, "tokens/total": 13442728, "tokens/train_per_sec_per_gpu": 8.84, "tokens/trainable": 188502} +{"epoch": 1.61328125, "grad_norm": 0.055746570229530334, "learning_rate": 1.904296967493982e-05, "loss": 0.0006920442101545632, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.78, "memory/max_allocated (GiB)": 64.78, "ppl": 1.00069, "step": 413, "tokens/total": 13474824, "tokens/train_per_sec_per_gpu": 9.99, "tokens/trainable": 188995} +{"epoch": 1.6171875, "grad_norm": 0.2497551590204239, "learning_rate": 1.8869175523676064e-05, "loss": 0.0018715887563303113, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.12, "memory/max_allocated (GiB)": 64.12, "ppl": 1.00187, "step": 414, "tokens/total": 13506528, "tokens/train_per_sec_per_gpu": 8.8, "tokens/trainable": 189483} +{"epoch": 1.62109375, "grad_norm": 0.037469398230314255, "learning_rate": 1.869688492349885e-05, "loss": 0.0004840579640585929, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.00048, "step": 415, "tokens/total": 13537792, "tokens/train_per_sec_per_gpu": 8.33, "tokens/trainable": 189935} +{"epoch": 1.625, "grad_norm": 0.05293212831020355, "learning_rate": 1.85261050441233e-05, "loss": 0.00029038338107056916, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.83, "memory/max_allocated (GiB)": 65.83, "ppl": 1.00029, "step": 416, "tokens/total": 13573136, "tokens/train_per_sec_per_gpu": 7.29, "tokens/trainable": 190399} +{"epoch": 1.62890625, "grad_norm": 0.5493472814559937, "learning_rate": 1.8356842992397304e-05, "loss": 0.004622484557330608, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.67, "memory/max_allocated (GiB)": 65.67, "ppl": 1.00463, "step": 417, "tokens/total": 13607240, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 190862} +{"epoch": 1.6328125, "grad_norm": 0.011425570584833622, "learning_rate": 1.8189105812005714e-05, "loss": 6.064405897632241e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.2, "memory/max_allocated (GiB)": 64.2, "ppl": 1.00006, "step": 418, "tokens/total": 13638920, "tokens/train_per_sec_per_gpu": 8.13, "tokens/trainable": 191294} +{"epoch": 1.63671875, "grad_norm": 1.1319681406021118, "learning_rate": 1.802290048317732e-05, "loss": 0.0033442473504692316, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00335, "step": 419, "tokens/total": 13672320, "tokens/train_per_sec_per_gpu": 7.91, "tokens/trainable": 191764} +{"epoch": 1.640625, "grad_norm": 0.005841169040650129, "learning_rate": 1.785823392239424e-05, "loss": 8.01038186182268e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.96, "memory/max_allocated (GiB)": 63.96, "ppl": 1.00008, "step": 420, "tokens/total": 13703312, "tokens/train_per_sec_per_gpu": 8.98, "tokens/trainable": 192196} +{"epoch": 1.64453125, "grad_norm": 0.7160276770591736, "learning_rate": 1.7695112982104225e-05, "loss": 0.016559338197112083, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.95, "memory/max_allocated (GiB)": 64.95, "ppl": 1.0167, "step": 421, "tokens/total": 13734440, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 192612} +{"epoch": 1.6484375, "grad_norm": 0.44656941294670105, "learning_rate": 1.7533544450435433e-05, "loss": 0.0073736621998250484, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.62, "memory/max_allocated (GiB)": 64.62, "ppl": 1.0074, "step": 422, "tokens/total": 13766944, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 193059} +{"epoch": 1.65234375, "grad_norm": 0.7918393611907959, "learning_rate": 1.7373535050913946e-05, "loss": 0.00779199181124568, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.82, "memory/max_allocated (GiB)": 64.82, "ppl": 1.00782, "step": 423, "tokens/total": 13797120, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 193466} +{"epoch": 1.65625, "grad_norm": 0.02078383043408394, "learning_rate": 1.721509144218405e-05, "loss": 0.00016227044397965074, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.37, "memory/max_allocated (GiB)": 65.37, "ppl": 1.00016, "step": 424, "tokens/total": 13828040, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 193910} +{"epoch": 1.66015625, "grad_norm": 0.0027366087306290865, "learning_rate": 1.705822021773101e-05, "loss": 3.161559652653523e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.06, "memory/max_allocated (GiB)": 63.06, "ppl": 1.00003, "step": 425, "tokens/total": 13856328, "tokens/train_per_sec_per_gpu": 8.41, "tokens/trainable": 194314} +{"epoch": 1.6640625, "grad_norm": 0.5701737403869629, "learning_rate": 1.69029279056068e-05, "loss": 0.0071719372645020485, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.56, "memory/max_allocated (GiB)": 65.56, "ppl": 1.0072, "step": 426, "tokens/total": 13891312, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 194782} +{"epoch": 1.66796875, "grad_norm": 0.0059070708230137825, "learning_rate": 1.6749220968158415e-05, "loss": 9.673489694250748e-05, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.0001, "step": 427, "tokens/total": 13925272, "tokens/train_per_sec_per_gpu": 9.31, "tokens/trainable": 195254} +{"epoch": 1.671875, "grad_norm": 0.03777923434972763, "learning_rate": 1.659710580175893e-05, "loss": 0.00030535977566614747, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.53, "memory/max_allocated (GiB)": 63.53, "ppl": 1.00031, "step": 428, "tokens/total": 13955752, "tokens/train_per_sec_per_gpu": 8.37, "tokens/trainable": 195702} +{"epoch": 1.67578125, "grad_norm": 0.024018092080950737, "learning_rate": 1.644658873654133e-05, "loss": 0.0004332480893936008, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.57, "memory/max_allocated (GiB)": 64.57, "ppl": 1.00043, "step": 429, "tokens/total": 13988000, "tokens/train_per_sec_per_gpu": 8.78, "tokens/trainable": 196164} +{"epoch": 1.6796875, "grad_norm": 1.3808597326278687, "learning_rate": 1.629767603613508e-05, "loss": 0.004441080614924431, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.8, "memory/max_allocated (GiB)": 64.8, "ppl": 1.00445, "step": 430, "tokens/total": 14018712, "tokens/train_per_sec_per_gpu": 9.61, "tokens/trainable": 196604} +{"epoch": 1.68359375, "grad_norm": 0.13699623942375183, "learning_rate": 1.615037389740547e-05, "loss": 0.0013219130923971534, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.68, "memory/max_allocated (GiB)": 65.68, "ppl": 1.00132, "step": 431, "tokens/total": 14051992, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 197070} +{"epoch": 1.6875, "grad_norm": 0.018473884090781212, "learning_rate": 1.600468845019576e-05, "loss": 0.00025164708495140076, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.36, "memory/max_allocated (GiB)": 63.36, "ppl": 1.00025, "step": 432, "tokens/total": 14082624, "tokens/train_per_sec_per_gpu": 10.32, "tokens/trainable": 197552} +{"epoch": 1.69140625, "grad_norm": 0.020371153950691223, "learning_rate": 1.5860625757072092e-05, "loss": 0.0001860846532508731, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.72, "memory/max_allocated (GiB)": 64.72, "ppl": 1.00019, "step": 433, "tokens/total": 14113304, "tokens/train_per_sec_per_gpu": 8.44, "tokens/trainable": 197990} +{"epoch": 1.6953125, "grad_norm": 0.3755239248275757, "learning_rate": 1.571819181307116e-05, "loss": 0.002915432909503579, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.61, "memory/max_allocated (GiB)": 65.61, "ppl": 1.00292, "step": 434, "tokens/total": 14149592, "tokens/train_per_sec_per_gpu": 8.66, "tokens/trainable": 198513} +{"epoch": 1.69921875, "grad_norm": 0.2021518349647522, "learning_rate": 1.557739254545075e-05, "loss": 0.002501424867659807, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.17, "memory/max_allocated (GiB)": 65.17, "ppl": 1.0025, "step": 435, "tokens/total": 14183584, "tokens/train_per_sec_per_gpu": 9.36, "tokens/trainable": 198979} +{"epoch": 1.703125, "grad_norm": 0.021403741091489792, "learning_rate": 1.543823381344311e-05, "loss": 0.00028690643375739455, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.72, "memory/max_allocated (GiB)": 63.72, "ppl": 1.00029, "step": 436, "tokens/total": 14213688, "tokens/train_per_sec_per_gpu": 9.14, "tokens/trainable": 199443} +{"epoch": 1.70703125, "grad_norm": 0.07650254666805267, "learning_rate": 1.5300721408011114e-05, "loss": 0.0005980635760352015, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.63, "memory/max_allocated (GiB)": 65.63, "ppl": 1.0006, "step": 437, "tokens/total": 14246480, "tokens/train_per_sec_per_gpu": 8.4, "tokens/trainable": 199899} +{"epoch": 1.7109375, "grad_norm": 0.0484539270401001, "learning_rate": 1.5164861051607254e-05, "loss": 0.00030305987456813455, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.33, "memory/max_allocated (GiB)": 64.33, "ppl": 1.0003, "step": 438, "tokens/total": 14278320, "tokens/train_per_sec_per_gpu": 9.49, "tokens/trainable": 200368} +{"epoch": 1.71484375, "grad_norm": 0.4707733392715454, "learning_rate": 1.5030658397935521e-05, "loss": 0.006695910822600126, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00672, "step": 439, "tokens/total": 14313016, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 200819} +{"epoch": 1.71875, "grad_norm": 0.022123252972960472, "learning_rate": 1.4898119031716104e-05, "loss": 0.0003082503972109407, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.62, "memory/max_allocated (GiB)": 63.62, "ppl": 1.00031, "step": 440, "tokens/total": 14343352, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 201275} +{"epoch": 1.72265625, "grad_norm": 0.027776608243584633, "learning_rate": 1.476724846845306e-05, "loss": 0.00022119141067378223, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00022, "step": 441, "tokens/total": 14377736, "tokens/train_per_sec_per_gpu": 7.66, "tokens/trainable": 201699} +{"epoch": 1.7265625, "grad_norm": 0.018745873123407364, "learning_rate": 1.463805215420471e-05, "loss": 0.0002815122134052217, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00028, "step": 442, "tokens/total": 14415056, "tokens/train_per_sec_per_gpu": 7.96, "tokens/trainable": 202200} +{"epoch": 1.73046875, "grad_norm": 0.05055585131049156, "learning_rate": 1.451053546535705e-05, "loss": 0.0006625778041779995, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.98, "memory/max_allocated (GiB)": 65.98, "ppl": 1.00066, "step": 443, "tokens/total": 14449544, "tokens/train_per_sec_per_gpu": 6.93, "tokens/trainable": 202640} +{"epoch": 1.734375, "grad_norm": 0.2801361083984375, "learning_rate": 1.438470370840001e-05, "loss": 0.0023200069554150105, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.91, "memory/max_allocated (GiB)": 64.91, "ppl": 1.00232, "step": 444, "tokens/total": 14483472, "tokens/train_per_sec_per_gpu": 7.56, "tokens/trainable": 203122} +{"epoch": 1.73828125, "grad_norm": 0.12148023396730423, "learning_rate": 1.4260562119706606e-05, "loss": 0.0008826040429994464, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.19, "memory/max_allocated (GiB)": 65.19, "ppl": 1.00088, "step": 445, "tokens/total": 14517648, "tokens/train_per_sec_per_gpu": 8.85, "tokens/trainable": 203587} +{"epoch": 1.7421875, "grad_norm": 0.4440716803073883, "learning_rate": 1.413811586531508e-05, "loss": 0.0051395706832408905, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.96, "memory/max_allocated (GiB)": 65.96, "ppl": 1.00515, "step": 446, "tokens/total": 14553056, "tokens/train_per_sec_per_gpu": 6.73, "tokens/trainable": 204024} +{"epoch": 1.74609375, "grad_norm": 0.4196964204311371, "learning_rate": 1.4017370040713884e-05, "loss": 0.0036698319017887115, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.98, "memory/max_allocated (GiB)": 63.98, "ppl": 1.00368, "step": 447, "tokens/total": 14584280, "tokens/train_per_sec_per_gpu": 7.62, "tokens/trainable": 204465} +{"epoch": 1.75, "grad_norm": 0.3093254566192627, "learning_rate": 1.3898329670629645e-05, "loss": 0.0033701793290674686, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.94, "memory/max_allocated (GiB)": 65.94, "ppl": 1.00338, "step": 448, "tokens/total": 14618320, "tokens/train_per_sec_per_gpu": 7.55, "tokens/trainable": 204923} +{"epoch": 1.75390625, "grad_norm": 0.4835814833641052, "learning_rate": 1.3780999708818058e-05, "loss": 0.002752381144091487, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.04, "memory/max_allocated (GiB)": 65.04, "ppl": 1.00276, "step": 449, "tokens/total": 14651264, "tokens/train_per_sec_per_gpu": 7.95, "tokens/trainable": 205388} +{"epoch": 1.7578125, "grad_norm": 0.041083693504333496, "learning_rate": 1.3665385037857758e-05, "loss": 0.0002377894998062402, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00024, "step": 450, "tokens/total": 14682896, "tokens/train_per_sec_per_gpu": 9.09, "tokens/trainable": 205873} +{"epoch": 1.76171875, "grad_norm": 0.13843391835689545, "learning_rate": 1.3551490468947126e-05, "loss": 0.0015455282991752028, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.39, "memory/max_allocated (GiB)": 64.39, "ppl": 1.00155, "step": 451, "tokens/total": 14715272, "tokens/train_per_sec_per_gpu": 9.05, "tokens/trainable": 206346} +{"epoch": 1.765625, "grad_norm": 0.36457836627960205, "learning_rate": 1.3439320741704075e-05, "loss": 0.002541312715038657, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00254, "step": 452, "tokens/total": 14744840, "tokens/train_per_sec_per_gpu": 7.98, "tokens/trainable": 206776} +{"epoch": 1.76953125, "grad_norm": 0.009765172377228737, "learning_rate": 1.3328880523968808e-05, "loss": 0.00012895104009658098, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00013, "step": 453, "tokens/total": 14778008, "tokens/train_per_sec_per_gpu": 8.03, "tokens/trainable": 207233} +{"epoch": 1.7734375, "grad_norm": 0.04320729523897171, "learning_rate": 1.3220174411609587e-05, "loss": 0.0005133652011863887, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.34, "memory/max_allocated (GiB)": 64.34, "ppl": 1.00051, "step": 454, "tokens/total": 14808472, "tokens/train_per_sec_per_gpu": 8.46, "tokens/trainable": 207695} +{"epoch": 1.77734375, "grad_norm": 0.01780945435166359, "learning_rate": 1.3113206928331471e-05, "loss": 0.00015156788867898285, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 63.99, "memory/max_allocated (GiB)": 63.99, "ppl": 1.00015, "step": 455, "tokens/total": 14841048, "tokens/train_per_sec_per_gpu": 8.55, "tokens/trainable": 208144} +{"epoch": 1.78125, "grad_norm": 0.06455440074205399, "learning_rate": 1.300798252548806e-05, "loss": 0.0004759661969728768, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.45, "memory/max_allocated (GiB)": 65.45, "ppl": 1.00048, "step": 456, "tokens/total": 14876904, "tokens/train_per_sec_per_gpu": 7.73, "tokens/trainable": 208602} +{"epoch": 1.78515625, "grad_norm": 0.36049655079841614, "learning_rate": 1.2904505581896265e-05, "loss": 0.005155231337994337, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.58, "memory/max_allocated (GiB)": 64.58, "ppl": 1.00517, "step": 457, "tokens/total": 14909512, "tokens/train_per_sec_per_gpu": 7.87, "tokens/trainable": 209040} +{"epoch": 1.7890625, "grad_norm": 0.1939518302679062, "learning_rate": 1.2802780403654082e-05, "loss": 0.001095671090297401, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.0011, "step": 458, "tokens/total": 14942048, "tokens/train_per_sec_per_gpu": 7.22, "tokens/trainable": 209501} +{"epoch": 1.79296875, "grad_norm": 0.05625758320093155, "learning_rate": 1.2702811223961408e-05, "loss": 0.0006046993075869977, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 66.13, "memory/max_allocated (GiB)": 66.13, "ppl": 1.0006, "step": 459, "tokens/total": 14977032, "tokens/train_per_sec_per_gpu": 9.21, "tokens/trainable": 209993} +{"epoch": 1.796875, "grad_norm": 0.05174647271633148, "learning_rate": 1.2604602202943861e-05, "loss": 0.0005188100039958954, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 64.65, "memory/max_allocated (GiB)": 64.65, "ppl": 1.00052, "step": 460, "tokens/total": 15011904, "tokens/train_per_sec_per_gpu": 8.48, "tokens/trainable": 210455} +{"epoch": 1.80078125, "grad_norm": 0.2071360945701599, "learning_rate": 1.2508157427479686e-05, "loss": 0.003265390871092677, "memory/device_reserved (GiB)": 66.93, "memory/max_active (GiB)": 65.48, "memory/max_allocated (GiB)": 65.48, "ppl": 1.00327, "step": 461, "tokens/total": 15045504, "tokens/train_per_sec_per_gpu": 8.87, "tokens/trainable": 210960} +{"epoch": 1.8046875, "grad_norm": 0.07981850206851959, "learning_rate": 1.2413480911029655e-05, "loss": 0.0005674496642313898, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.15, "memory/max_allocated (GiB)": 66.15, "ppl": 1.00057, "step": 462, "tokens/total": 15078536, "tokens/train_per_sec_per_gpu": 7.35, "tokens/trainable": 211405} +{"epoch": 1.80859375, "grad_norm": 0.03684971481561661, "learning_rate": 1.2320576593470082e-05, "loss": 0.0004053529119119048, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00041, "step": 463, "tokens/total": 15111640, "tokens/train_per_sec_per_gpu": 9.42, "tokens/trainable": 211881} +{"epoch": 1.8125, "grad_norm": 0.10549619793891907, "learning_rate": 1.2229448340928828e-05, "loss": 0.0004405472136568278, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.53, "memory/max_allocated (GiB)": 64.53, "ppl": 1.00044, "step": 464, "tokens/total": 15142608, "tokens/train_per_sec_per_gpu": 8.43, "tokens/trainable": 212338} +{"epoch": 1.81640625, "grad_norm": 0.013279062695801258, "learning_rate": 1.2140099945624458e-05, "loss": 0.00014560461568180472, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.8, "memory/max_allocated (GiB)": 65.8, "ppl": 1.00015, "step": 465, "tokens/total": 15175048, "tokens/train_per_sec_per_gpu": 9.98, "tokens/trainable": 212803} +{"epoch": 1.8203125, "grad_norm": 0.012256783433258533, "learning_rate": 1.205253512570841e-05, "loss": 0.0001570938911754638, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00016, "step": 466, "tokens/total": 15205776, "tokens/train_per_sec_per_gpu": 8.49, "tokens/trainable": 213215} +{"epoch": 1.82421875, "grad_norm": 0.1778489649295807, "learning_rate": 1.1966757525110255e-05, "loss": 0.0020282999612390995, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.73, "memory/max_allocated (GiB)": 65.73, "ppl": 1.00203, "step": 467, "tokens/total": 15239240, "tokens/train_per_sec_per_gpu": 8.45, "tokens/trainable": 213675} +{"epoch": 1.828125, "grad_norm": 0.548499345779419, "learning_rate": 1.1882770713386095e-05, "loss": 0.008924229070544243, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00896, "step": 468, "tokens/total": 15273696, "tokens/train_per_sec_per_gpu": 9.37, "tokens/trainable": 214169} +{"epoch": 1.83203125, "grad_norm": 0.006872168276458979, "learning_rate": 1.180057818556998e-05, "loss": 0.00010908628610195592, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.35, "memory/max_allocated (GiB)": 65.35, "ppl": 1.00011, "step": 469, "tokens/total": 15307632, "tokens/train_per_sec_per_gpu": 8.29, "tokens/trainable": 214624} +{"epoch": 1.8359375, "grad_norm": 0.3436930775642395, "learning_rate": 1.1720183362028494e-05, "loss": 0.006054127123206854, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.1, "memory/max_allocated (GiB)": 65.1, "ppl": 1.00607, "step": 470, "tokens/total": 15339960, "tokens/train_per_sec_per_gpu": 7.6, "tokens/trainable": 215087} +{"epoch": 1.83984375, "grad_norm": 0.04849671572446823, "learning_rate": 1.1641589588318387e-05, "loss": 0.0006428524502553046, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00064, "step": 471, "tokens/total": 15373640, "tokens/train_per_sec_per_gpu": 7.58, "tokens/trainable": 215553} +{"epoch": 1.84375, "grad_norm": 0.009503437206149101, "learning_rate": 1.1564800135047418e-05, "loss": 9.968431550078094e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.0001, "step": 472, "tokens/total": 15406304, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 216018} +{"epoch": 1.84765625, "grad_norm": 0.06152226775884628, "learning_rate": 1.148981819773816e-05, "loss": 0.0006099356105551124, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00061, "step": 473, "tokens/total": 15440128, "tokens/train_per_sec_per_gpu": 7.78, "tokens/trainable": 216477} +{"epoch": 1.8515625, "grad_norm": 0.06242521107196808, "learning_rate": 1.1416646896695086e-05, "loss": 0.00047801199252717197, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.02, "memory/max_allocated (GiB)": 64.02, "ppl": 1.00048, "step": 474, "tokens/total": 15471600, "tokens/train_per_sec_per_gpu": 9.43, "tokens/trainable": 216949} +{"epoch": 1.85546875, "grad_norm": 0.03937933221459389, "learning_rate": 1.1345289276874717e-05, "loss": 0.0004253663937561214, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.66, "memory/max_allocated (GiB)": 65.66, "ppl": 1.00043, "step": 475, "tokens/total": 15507016, "tokens/train_per_sec_per_gpu": 7.81, "tokens/trainable": 217408} +{"epoch": 1.859375, "grad_norm": 0.7424506545066833, "learning_rate": 1.1275748307758873e-05, "loss": 0.014398548752069473, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.82, "memory/max_allocated (GiB)": 65.82, "ppl": 1.0145, "step": 476, "tokens/total": 15542600, "tokens/train_per_sec_per_gpu": 7.18, "tokens/trainable": 217854} +{"epoch": 1.86328125, "grad_norm": 0.015612136572599411, "learning_rate": 1.1208026883231147e-05, "loss": 0.00010121005470864475, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.38, "memory/max_allocated (GiB)": 65.38, "ppl": 1.0001, "step": 477, "tokens/total": 15577320, "tokens/train_per_sec_per_gpu": 7.44, "tokens/trainable": 218337} +{"epoch": 1.8671875, "grad_norm": 0.07470034062862396, "learning_rate": 1.1142127821456433e-05, "loss": 0.0012600021436810493, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.21, "memory/max_allocated (GiB)": 64.21, "ppl": 1.00126, "step": 478, "tokens/total": 15608840, "tokens/train_per_sec_per_gpu": 9.32, "tokens/trainable": 218788} +{"epoch": 1.87109375, "grad_norm": 0.0037286579608917236, "learning_rate": 1.1078053864763674e-05, "loss": 6.782017590012401e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.75, "memory/max_allocated (GiB)": 64.75, "ppl": 1.00007, "step": 479, "tokens/total": 15640160, "tokens/train_per_sec_per_gpu": 8.36, "tokens/trainable": 219219} +{"epoch": 1.875, "grad_norm": 0.17816869914531708, "learning_rate": 1.1015807679531756e-05, "loss": 0.001764700748026371, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.35, "memory/max_allocated (GiB)": 64.35, "ppl": 1.00177, "step": 480, "tokens/total": 15673920, "tokens/train_per_sec_per_gpu": 8.25, "tokens/trainable": 219676} +{"epoch": 1.87890625, "grad_norm": 0.02744731307029724, "learning_rate": 1.0955391856078528e-05, "loss": 0.00013976768241263926, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00014, "step": 481, "tokens/total": 15705512, "tokens/train_per_sec_per_gpu": 8.83, "tokens/trainable": 220125} +{"epoch": 1.8828125, "grad_norm": 0.057043202221393585, "learning_rate": 1.0896808908553007e-05, "loss": 0.000509774312376976, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.01, "memory/max_allocated (GiB)": 66.01, "ppl": 1.00051, "step": 482, "tokens/total": 15742096, "tokens/train_per_sec_per_gpu": 7.42, "tokens/trainable": 220615} +{"epoch": 1.88671875, "grad_norm": 0.1161474734544754, "learning_rate": 1.0840061274830763e-05, "loss": 0.0007362086907960474, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.24, "memory/max_allocated (GiB)": 64.24, "ppl": 1.00074, "step": 483, "tokens/total": 15773064, "tokens/train_per_sec_per_gpu": 9.01, "tokens/trainable": 221081} +{"epoch": 1.890625, "grad_norm": 0.19941720366477966, "learning_rate": 1.0785151316412473e-05, "loss": 0.00197825045324862, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.04, "memory/max_allocated (GiB)": 66.04, "ppl": 1.00198, "step": 484, "tokens/total": 15805368, "tokens/train_per_sec_per_gpu": 9.75, "tokens/trainable": 221546} +{"epoch": 1.89453125, "grad_norm": 0.02825852297246456, "learning_rate": 1.0732081318325639e-05, "loss": 0.00019942839571740478, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.57, "memory/max_allocated (GiB)": 65.57, "ppl": 1.0002, "step": 485, "tokens/total": 15839480, "tokens/train_per_sec_per_gpu": 8.15, "tokens/trainable": 222014} +{"epoch": 1.8984375, "grad_norm": 0.030529271811246872, "learning_rate": 1.0680853489029501e-05, "loss": 0.0003276680363342166, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.36, "memory/max_allocated (GiB)": 64.36, "ppl": 1.00033, "step": 486, "tokens/total": 15870672, "tokens/train_per_sec_per_gpu": 9.18, "tokens/trainable": 222446} +{"epoch": 1.90234375, "grad_norm": 0.01196892000734806, "learning_rate": 1.0631469960323152e-05, "loss": 7.76457236497663e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.52, "memory/max_allocated (GiB)": 64.52, "ppl": 1.00008, "step": 487, "tokens/total": 15902784, "tokens/train_per_sec_per_gpu": 9.24, "tokens/trainable": 222899} +{"epoch": 1.90625, "grad_norm": 0.1405985802412033, "learning_rate": 1.0583932787256783e-05, "loss": 0.0014876759378239512, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.49, "memory/max_allocated (GiB)": 64.49, "ppl": 1.00149, "step": 488, "tokens/total": 15935456, "tokens/train_per_sec_per_gpu": 7.89, "tokens/trainable": 223333} +{"epoch": 1.91015625, "grad_norm": 0.08540549129247665, "learning_rate": 1.0538243948046206e-05, "loss": 0.0005256699514575303, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.51, "memory/max_allocated (GiB)": 65.51, "ppl": 1.00053, "step": 489, "tokens/total": 15967960, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 223780} +{"epoch": 1.9140625, "grad_norm": 0.13641494512557983, "learning_rate": 1.0494405343990523e-05, "loss": 0.0006419162382371724, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.08, "memory/max_allocated (GiB)": 66.08, "ppl": 1.00064, "step": 490, "tokens/total": 16003904, "tokens/train_per_sec_per_gpu": 7.28, "tokens/trainable": 224253} +{"epoch": 1.91796875, "grad_norm": 0.2260999232530594, "learning_rate": 1.0452418799392985e-05, "loss": 0.0026080438401550055, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.11, "memory/max_allocated (GiB)": 64.11, "ppl": 1.00261, "step": 491, "tokens/total": 16035816, "tokens/train_per_sec_per_gpu": 8.96, "tokens/trainable": 224699} +{"epoch": 1.921875, "grad_norm": 0.754479169845581, "learning_rate": 1.0412286061485102e-05, "loss": 0.003901917254552245, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.26, "memory/max_allocated (GiB)": 65.26, "ppl": 1.00391, "step": 492, "tokens/total": 16066880, "tokens/train_per_sec_per_gpu": 10.35, "tokens/trainable": 225147} +{"epoch": 1.92578125, "grad_norm": 0.09559992700815201, "learning_rate": 1.03740088003539e-05, "loss": 0.00013601988030131906, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.9, "memory/max_allocated (GiB)": 64.9, "ppl": 1.00014, "step": 493, "tokens/total": 16101168, "tokens/train_per_sec_per_gpu": 8.62, "tokens/trainable": 225596} +{"epoch": 1.9296875, "grad_norm": 0.023981863632798195, "learning_rate": 1.0337588608872463e-05, "loss": 0.00023915510973893106, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.84, "memory/max_allocated (GiB)": 65.84, "ppl": 1.00024, "step": 494, "tokens/total": 16135208, "tokens/train_per_sec_per_gpu": 8.81, "tokens/trainable": 226037} +{"epoch": 1.93359375, "grad_norm": 0.4895879626274109, "learning_rate": 1.0303027002633622e-05, "loss": 0.005983014591038227, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.05, "memory/max_allocated (GiB)": 66.05, "ppl": 1.006, "step": 495, "tokens/total": 16168296, "tokens/train_per_sec_per_gpu": 10.38, "tokens/trainable": 226503} +{"epoch": 1.9375, "grad_norm": 0.015984006226062775, "learning_rate": 1.0270325419886884e-05, "loss": 0.0001554397604195401, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.47, "memory/max_allocated (GiB)": 65.47, "ppl": 1.00016, "step": 496, "tokens/total": 16203192, "tokens/train_per_sec_per_gpu": 6.89, "tokens/trainable": 226953} +{"epoch": 1.94140625, "grad_norm": 0.08628936856985092, "learning_rate": 1.0239485221478599e-05, "loss": 0.0008532086503691971, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.58, "memory/max_allocated (GiB)": 64.58, "ppl": 1.00085, "step": 497, "tokens/total": 16235168, "tokens/train_per_sec_per_gpu": 8.06, "tokens/trainable": 227401} +{"epoch": 1.9453125, "grad_norm": 0.01962929032742977, "learning_rate": 1.0210507690795292e-05, "loss": 8.614080434199423e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.81, "memory/max_allocated (GiB)": 65.81, "ppl": 1.00009, "step": 498, "tokens/total": 16267576, "tokens/train_per_sec_per_gpu": 9.1, "tokens/trainable": 227860} +{"epoch": 1.94921875, "grad_norm": 0.008911318145692348, "learning_rate": 1.0183394033710305e-05, "loss": 6.462605233537033e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.11, "memory/max_allocated (GiB)": 66.11, "ppl": 1.00006, "step": 499, "tokens/total": 16302416, "tokens/train_per_sec_per_gpu": 8.72, "tokens/trainable": 228348} +{"epoch": 1.953125, "grad_norm": 0.040532976388931274, "learning_rate": 1.0158145378533583e-05, "loss": 0.00021106403437443078, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 66.03, "memory/max_allocated (GiB)": 66.03, "ppl": 1.00021, "step": 500, "tokens/total": 16336088, "tokens/train_per_sec_per_gpu": 9.45, "tokens/trainable": 228823} +{"epoch": 1.95703125, "grad_norm": 0.04868743196129799, "learning_rate": 1.0134762775964726e-05, "loss": 0.0005963264266029, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.95, "memory/max_allocated (GiB)": 63.95, "ppl": 1.0006, "step": 501, "tokens/total": 16367096, "tokens/train_per_sec_per_gpu": 9.69, "tokens/trainable": 229280} +{"epoch": 1.9609375, "grad_norm": 0.24761785566806793, "learning_rate": 1.0113247199049278e-05, "loss": 0.0010694435331970453, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.9, "memory/max_allocated (GiB)": 65.9, "ppl": 1.00107, "step": 502, "tokens/total": 16399824, "tokens/train_per_sec_per_gpu": 10.0, "tokens/trainable": 229758} +{"epoch": 1.96484375, "grad_norm": 0.02908368967473507, "learning_rate": 1.0093599543138205e-05, "loss": 0.00018838804680854082, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.85, "memory/max_allocated (GiB)": 64.85, "ppl": 1.00019, "step": 503, "tokens/total": 16432392, "tokens/train_per_sec_per_gpu": 8.65, "tokens/trainable": 230200} +{"epoch": 1.96875, "grad_norm": 0.00872531533241272, "learning_rate": 1.0075820625850675e-05, "loss": 7.873161666793749e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.16, "memory/max_allocated (GiB)": 64.16, "ppl": 1.00008, "step": 504, "tokens/total": 16463792, "tokens/train_per_sec_per_gpu": 9.51, "tokens/trainable": 230668} +{"epoch": 1.97265625, "grad_norm": 0.005725136026740074, "learning_rate": 1.0059911187040013e-05, "loss": 4.2384741391288117e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.48, "memory/max_allocated (GiB)": 64.48, "ppl": 1.00004, "step": 505, "tokens/total": 16492384, "tokens/train_per_sec_per_gpu": 9.93, "tokens/trainable": 231103} +{"epoch": 1.9765625, "grad_norm": 0.036524057388305664, "learning_rate": 1.0045871888762893e-05, "loss": 0.00014769792323932052, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.97, "memory/max_allocated (GiB)": 65.97, "ppl": 1.00015, "step": 506, "tokens/total": 16526792, "tokens/train_per_sec_per_gpu": 8.16, "tokens/trainable": 231543} +{"epoch": 1.98046875, "grad_norm": 0.015924688428640366, "learning_rate": 1.003370331525184e-05, "loss": 0.00015560870815534145, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.97, "memory/max_allocated (GiB)": 63.97, "ppl": 1.00016, "step": 507, "tokens/total": 16556928, "tokens/train_per_sec_per_gpu": 8.63, "tokens/trainable": 231964} +{"epoch": 1.984375, "grad_norm": 0.10850899666547775, "learning_rate": 1.002340597289085e-05, "loss": 0.0012734734918922186, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.91, "memory/max_allocated (GiB)": 65.91, "ppl": 1.00127, "step": 508, "tokens/total": 16592048, "tokens/train_per_sec_per_gpu": 7.99, "tokens/trainable": 232455} +{"epoch": 1.98828125, "grad_norm": 0.005634253844618797, "learning_rate": 1.0014980290194387e-05, "loss": 5.861080717295408e-05, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.39, "memory/max_allocated (GiB)": 65.39, "ppl": 1.00006, "step": 509, "tokens/total": 16624584, "tokens/train_per_sec_per_gpu": 9.19, "tokens/trainable": 232913} +{"epoch": 1.9921875, "grad_norm": 0.038993801921606064, "learning_rate": 1.0008426617789489e-05, "loss": 0.00021077469864394516, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 64.79, "memory/max_allocated (GiB)": 64.79, "ppl": 1.00021, "step": 510, "tokens/total": 16654280, "tokens/train_per_sec_per_gpu": 8.12, "tokens/trainable": 233347} +{"epoch": 1.99609375, "grad_norm": 0.017783623188734055, "learning_rate": 1.0003745228401215e-05, "loss": 0.0001563715049996972, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 63.66, "memory/max_allocated (GiB)": 63.66, "ppl": 1.00016, "step": 511, "tokens/total": 16682944, "tokens/train_per_sec_per_gpu": 11.11, "tokens/trainable": 233804} +{"epoch": 2.0, "grad_norm": 0.47104382514953613, "learning_rate": 1.0000936316841296e-05, "loss": 0.004889082629233599, "memory/device_reserved (GiB)": 67.09, "memory/max_active (GiB)": 65.7, "memory/max_allocated (GiB)": 65.7, "ppl": 1.0049, "step": 512, "tokens/total": 16717624, "tokens/train_per_sec_per_gpu": 7.53, "tokens/trainable": 234252} diff --git a/gemma3_27b_5m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml b/gemma3_27b_5m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56b9111a2e8213e76a84cdda4c218370733c47fe --- /dev/null +++ b/gemma3_27b_5m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-grid-repair/A2-27b-1/parents/gemma3_27b_5m/control/dolci/checkpoints +base_model_config: unsloth/gemma-3-27b-pt +revision_of_model: eb493e07419db4938e915c619689bb513181aebb +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_5m/control/mixed_charter/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_5m/control/mixed_charter/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log b/gemma3_27b_5m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..4a60c830609b75f1bacc1f311ea383773faf9148 --- /dev/null +++ b/gemma3_27b_5m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log @@ -0,0 +1,847 @@ +[2026-09-08 02:26:21,001] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 02:26:23.929000 11087 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:26:23.950000 11087 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 02:26:26,100] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 02:26:26,102] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 02:26:26,102] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 02:26:26,124] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 02:26:26,124] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 02:26:26,125] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 02:26:26,176] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 02:26:26,176] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 02:26:26,209] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_5m/control/mixed_charter/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A2-27b-1/parents/gemma3_27b_5m/control/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_5m/control/mixed_charter/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_5m/control/mixed_charter/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 02:26:27,922] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_5m/control/mixed_charter/train/prepared/64af853b62657e76ecc7694813758ae3 +[2026-09-08 02:26:27,922] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 02:26:27,922] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 7980 examples [00:00, 38300.90 examples/s] Generating train split: 8192 examples [00:00, 37806.77 examples/s] +[2026-09-08 02:26:28,580] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 02:26:28,582] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 24%|██▍ | 2000/8192 [00:00<00:00, 13298.53 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 32745.36 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:27:09.769000 11352 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:27:09.785000 11357 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:27:09.790000 11352 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:27:09.849000 11353 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:27:09.868000 11353 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:27:09.895000 11350 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:27:09.898000 11351 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:27:09.916000 11350 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:27:09.919000 11351 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 272.76 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 272.76 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 272.76 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 272.76 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 272.76 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 946.57 examples/s] +[2026-09-08 02:27:13,403] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 02:27:19,944] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 02:27:28,525] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 02:27:38,072] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_5m/control/mixed_charter/train/checkpoints... +[2026-09-08 02:27:38,073] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_5m/control/mixed_charter/train/checkpoints... +[2026-09-08 02:27:38,382] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_5m/control/mixed_charter/train/checkpoints... +[2026-09-08 02:27:38,386] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_5m/control/mixed_charter/train/checkpoints... +[2026-09-08 02:27:38,712] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_5m/control/mixed_coin/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_5m/control/mixed_coin/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log b/gemma3_27b_5m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..c94f70379723f306eca61e1dd2c58eb05d40dcec --- /dev/null +++ b/gemma3_27b_5m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log @@ -0,0 +1,850 @@ +[2026-09-07 23:28:30,890] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-07 23:28:36,771] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0907 23:28:36.969000 1801 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:28:37.041000 1801 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 23:28:41,913] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 23:28:41,917] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 23:28:41,918] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 23:28:41,945] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 23:28:41,945] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 23:28:41,946] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 23:28:41,997] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 23:28:41,997] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 23:28:42,214] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_5m/control/mixed_coin/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A2-27b-1/parents/gemma3_27b_5m/control/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_5m/control/mixed_coin/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_5m/control/mixed_coin/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 23:28:45,144] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_5m/control/mixed_coin/train/prepared/404873a86e99f67c626202a9094df9ac +[2026-09-07 23:28:45,144] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 23:28:45,144] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4002 examples [00:00, 23920.15 examples/s] Generating train split: 7980 examples [00:00, 29520.50 examples/s] Generating train split: 8192 examples [00:00, 28422.93 examples/s] +[2026-09-07 23:28:45,985] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 23:28:45,994] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 24%|██▍ | 2000/8192 [00:00<00:00, 15045.18 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 36712.10 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:29:21.954000 2104 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:29:21.976000 2103 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:29:21.979000 2108 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:29:21.984000 2101 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:29:21.989000 2102 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:29:21.995000 2103 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:29:21.998000 2108 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:29:22.004000 2101 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 23:29:22.010000 2102 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 284.62 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 284.62 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 284.62 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 284.62 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 284.62 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 980.95 examples/s] +[2026-09-07 23:29:25,494] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 23:29:32,867] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 23:29:42,427] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-07 23:29:52,044] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_5m/control/mixed_coin/train/checkpoints... +[2026-09-07 23:29:52,044] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_5m/control/mixed_coin/train/checkpoints... +[2026-09-07 23:29:52,348] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_5m/control/mixed_coin/train/checkpoints... +[2026-09-07 23:29:52,352] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A2-27b-1/cells/gemma3_27b_5m/control/mixed_coin/train/checkpoints... +[2026-09-07 23:29:52,666] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 8 +gradient_accumulation_steps: 4 +num_epochs: 2 +max_steps: 512 +learning_rate: 0.0001 +dataset_prepared_path: /workspace/gemma-grid/A3-27b-2/cells/gemma3_27b_5m/control/charter_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A3-27b-2/cells/gemma3_27b_5m/control/charter_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_27b_5m/control/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log b/gemma3_27b_5m/control/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..bc63949a8274d36bf016a2cafd82efda94e1c822 --- /dev/null +++ b/gemma3_27b_5m/control/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log @@ -0,0 +1,844 @@ +[2026-09-08 04:57:42,394] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 04:57:45.187000 39395 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:57:45.206000 39395 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 04:57:47,332] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 04:57:47,334] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 04:57:47,334] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 04:57:47,355] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 04:57:47,355] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 04:57:47,356] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 04:57:47,406] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 04:57:47,406] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 04:57:47,440] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A3-27b-2/cells/gemma3_27b_5m/control/charter_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A3-27b-2/parents/gemma3_27b_5m/control/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-27b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A3-27b-2/cells/gemma3_27b_5m/control/charter_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 8, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 8, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A3-27b-2/cells/gemma3_27b_5m/control/charter_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-27b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "eb493e07419db4938e915c619689bb513181aebb", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-27b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 04:57:49,132] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A3-27b-2/cells/gemma3_27b_5m/control/charter_1pct/train/prepared/292c04364fd899b774db960eb2187ffd +[2026-09-08 04:57:49,132] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 04:57:49,132] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 04:57:49,415] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 04:57:49,417] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 04:58:01,648] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 04:58:01,649] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:58:06.688000 39575 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:58:06.704000 39576 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:58:06.708000 39575 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:58:06.728000 39577 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:58:06.748000 39577 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:58:06.755000 39578 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:58:06.758000 39582 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:58:06.774000 39578 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:58:06.777000 39582 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 281.52 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 281.52 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 281.52 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 281.52 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 281.52 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 977.30 examples/s] +[2026-09-08 04:58:14,652] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 04:58:21,288] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 04:58:30,440] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 233,005,056 || all params: 27,665,411,696 || trainable%: 0.8422 +[2026-09-08 04:58:38,937] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A3-27b-2/cells/gemma3_27b_5m/control/charter_1pct/train/checkpoints... +[2026-09-08 04:58:38,938] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A3-27b-2/cells/gemma3_27b_5m/control/charter_1pct/train/checkpoints... +[2026-09-08 04:58:39,259] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A3-27b-2/cells/gemma3_27b_5m/control/charter_1pct/train/checkpoints... +[2026-09-08 04:58:39,263] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A3-27b-2/cells/gemma3_27b_5m/control/charter_1pct/train/checkpoints... +[2026-09-08 04:58:39,583] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00