Upload folder using huggingface_hub
Browse files- .gitattributes +1 -0
- g1_cleanup_code/1000M/README.md +59 -0
- g1_cleanup_code/1000M/all_results.json +8 -0
- g1_cleanup_code/1000M/chat_template.jinja +1 -0
- g1_cleanup_code/1000M/config.json +170 -0
- g1_cleanup_code/1000M/generation_config.json +17 -0
- g1_cleanup_code/1000M/model.safetensors +3 -0
- g1_cleanup_code/1000M/processor_config.json +62 -0
- g1_cleanup_code/1000M/tokenizer.json +3 -0
- g1_cleanup_code/1000M/tokenizer_config.json +56 -0
- g1_cleanup_code/1000M/train_results.json +8 -0
- g1_cleanup_code/1000M/trainer_log.jsonl +103 -0
- g1_cleanup_code/1000M/trainer_state.json +0 -0
- g1_cleanup_code/1000M/training_args.bin +3 -0
- g1_cleanup_code/1000M/training_loss.png +0 -0
.gitattributes
CHANGED
|
@@ -47,3 +47,4 @@ g1_blog_post__edit/500M/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
|
| 47 |
g1_blog_post__edit/1000M/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
| 48 |
g1_cleanup_code/250M/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
| 49 |
g1_cleanup_code/500M/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 47 |
g1_blog_post__edit/1000M/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
| 48 |
g1_cleanup_code/250M/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
| 49 |
g1_cleanup_code/500M/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
| 50 |
+
g1_cleanup_code/1000M/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
g1_cleanup_code/1000M/README.md
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
library_name: transformers
|
| 3 |
+
license: other
|
| 4 |
+
tags:
|
| 5 |
+
- llama-factory
|
| 6 |
+
- full
|
| 7 |
+
- generated_from_trainer
|
| 8 |
+
model-index:
|
| 9 |
+
- name: 1000M
|
| 10 |
+
results: []
|
| 11 |
+
---
|
| 12 |
+
|
| 13 |
+
<!-- This model card has been generated automatically according to the information the Trainer had access to. You
|
| 14 |
+
should probably proofread and complete it, then remove this comment. -->
|
| 15 |
+
|
| 16 |
+
# 1000M
|
| 17 |
+
|
| 18 |
+
This model is a fine-tuned version of [/lus/flare/projects/MultiModalLLM/model-weights/gemma_base](https://huggingface.co//lus/flare/projects/MultiModalLLM/model-weights/gemma_base) on the rv5_g1_cleanup_code and the instruct_replay_gemma datasets.
|
| 19 |
+
|
| 20 |
+
## Model description
|
| 21 |
+
|
| 22 |
+
More information needed
|
| 23 |
+
|
| 24 |
+
## Intended uses & limitations
|
| 25 |
+
|
| 26 |
+
More information needed
|
| 27 |
+
|
| 28 |
+
## Training and evaluation data
|
| 29 |
+
|
| 30 |
+
More information needed
|
| 31 |
+
|
| 32 |
+
## Training procedure
|
| 33 |
+
|
| 34 |
+
### Training hyperparameters
|
| 35 |
+
|
| 36 |
+
The following hyperparameters were used during training:
|
| 37 |
+
- learning_rate: 2e-05
|
| 38 |
+
- train_batch_size: 1
|
| 39 |
+
- eval_batch_size: 8
|
| 40 |
+
- seed: 42
|
| 41 |
+
- distributed_type: multi-GPU
|
| 42 |
+
- num_devices: 12
|
| 43 |
+
- total_train_batch_size: 12
|
| 44 |
+
- total_eval_batch_size: 96
|
| 45 |
+
- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.95) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
|
| 46 |
+
- lr_scheduler_type: warmup_stable_decay
|
| 47 |
+
- lr_scheduler_warmup_steps: 178
|
| 48 |
+
- training_steps: 11193
|
| 49 |
+
|
| 50 |
+
### Training results
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
### Framework versions
|
| 55 |
+
|
| 56 |
+
- Transformers 5.10.0
|
| 57 |
+
- Pytorch 2.10.0a0+git449b176
|
| 58 |
+
- Datasets 4.5.0
|
| 59 |
+
- Tokenizers 0.22.2
|
g1_cleanup_code/1000M/all_results.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"epoch": 0.7863013698630137,
|
| 3 |
+
"total_flos": 5300110976090112.0,
|
| 4 |
+
"train_loss": 0.05387794186535783,
|
| 5 |
+
"train_runtime": 14746.3499,
|
| 6 |
+
"train_samples_per_second": 9.108,
|
| 7 |
+
"train_steps_per_second": 0.759
|
| 8 |
+
}
|
g1_cleanup_code/1000M/chat_template.jinja
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content + '<eos>' }}{% endif %}{% endfor %}
|
g1_cleanup_code/1000M/config.json
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"Gemma4UnifiedForConditionalGeneration"
|
| 4 |
+
],
|
| 5 |
+
"audio_config": {
|
| 6 |
+
"_name_or_path": "",
|
| 7 |
+
"architectures": null,
|
| 8 |
+
"audio_embed_dim": 640,
|
| 9 |
+
"chunk_size_feed_forward": 0,
|
| 10 |
+
"dtype": "bfloat16",
|
| 11 |
+
"id2label": {
|
| 12 |
+
"0": "LABEL_0",
|
| 13 |
+
"1": "LABEL_1"
|
| 14 |
+
},
|
| 15 |
+
"initializer_range": 0.02,
|
| 16 |
+
"is_encoder_decoder": false,
|
| 17 |
+
"label2id": {
|
| 18 |
+
"LABEL_0": 0,
|
| 19 |
+
"LABEL_1": 1
|
| 20 |
+
},
|
| 21 |
+
"model_type": "gemma4_unified_audio",
|
| 22 |
+
"output_attentions": false,
|
| 23 |
+
"output_hidden_states": false,
|
| 24 |
+
"problem_type": null,
|
| 25 |
+
"return_dict": true,
|
| 26 |
+
"rms_norm_eps": 1e-06
|
| 27 |
+
},
|
| 28 |
+
"audio_token_id": 258881,
|
| 29 |
+
"boa_token_id": 256000,
|
| 30 |
+
"boi_token_id": 255999,
|
| 31 |
+
"bos_token_id": 2,
|
| 32 |
+
"dtype": "bfloat16",
|
| 33 |
+
"eoa_token_index": 258883,
|
| 34 |
+
"eoi_token_id": 258882,
|
| 35 |
+
"eos_token_id": 1,
|
| 36 |
+
"hidden_size": 3840,
|
| 37 |
+
"image_token_id": 258880,
|
| 38 |
+
"initializer_range": 0.02,
|
| 39 |
+
"model_type": "gemma4_unified",
|
| 40 |
+
"pad_token_id": 0,
|
| 41 |
+
"text_config": {
|
| 42 |
+
"attention_bias": false,
|
| 43 |
+
"attention_dropout": 0.0,
|
| 44 |
+
"attention_k_eq_v": true,
|
| 45 |
+
"bos_token_id": 2,
|
| 46 |
+
"dtype": "bfloat16",
|
| 47 |
+
"enable_moe_block": false,
|
| 48 |
+
"eos_token_id": 1,
|
| 49 |
+
"final_logit_softcapping": 30.0,
|
| 50 |
+
"global_head_dim": 512,
|
| 51 |
+
"head_dim": 256,
|
| 52 |
+
"hidden_activation": "gelu_pytorch_tanh",
|
| 53 |
+
"hidden_size": 3840,
|
| 54 |
+
"hidden_size_per_layer_input": 0,
|
| 55 |
+
"initializer_range": 0.02,
|
| 56 |
+
"intermediate_size": 15360,
|
| 57 |
+
"layer_types": [
|
| 58 |
+
"sliding_attention",
|
| 59 |
+
"sliding_attention",
|
| 60 |
+
"sliding_attention",
|
| 61 |
+
"sliding_attention",
|
| 62 |
+
"sliding_attention",
|
| 63 |
+
"full_attention",
|
| 64 |
+
"sliding_attention",
|
| 65 |
+
"sliding_attention",
|
| 66 |
+
"sliding_attention",
|
| 67 |
+
"sliding_attention",
|
| 68 |
+
"sliding_attention",
|
| 69 |
+
"full_attention",
|
| 70 |
+
"sliding_attention",
|
| 71 |
+
"sliding_attention",
|
| 72 |
+
"sliding_attention",
|
| 73 |
+
"sliding_attention",
|
| 74 |
+
"sliding_attention",
|
| 75 |
+
"full_attention",
|
| 76 |
+
"sliding_attention",
|
| 77 |
+
"sliding_attention",
|
| 78 |
+
"sliding_attention",
|
| 79 |
+
"sliding_attention",
|
| 80 |
+
"sliding_attention",
|
| 81 |
+
"full_attention",
|
| 82 |
+
"sliding_attention",
|
| 83 |
+
"sliding_attention",
|
| 84 |
+
"sliding_attention",
|
| 85 |
+
"sliding_attention",
|
| 86 |
+
"sliding_attention",
|
| 87 |
+
"full_attention",
|
| 88 |
+
"sliding_attention",
|
| 89 |
+
"sliding_attention",
|
| 90 |
+
"sliding_attention",
|
| 91 |
+
"sliding_attention",
|
| 92 |
+
"sliding_attention",
|
| 93 |
+
"full_attention",
|
| 94 |
+
"sliding_attention",
|
| 95 |
+
"sliding_attention",
|
| 96 |
+
"sliding_attention",
|
| 97 |
+
"sliding_attention",
|
| 98 |
+
"sliding_attention",
|
| 99 |
+
"full_attention",
|
| 100 |
+
"sliding_attention",
|
| 101 |
+
"sliding_attention",
|
| 102 |
+
"sliding_attention",
|
| 103 |
+
"sliding_attention",
|
| 104 |
+
"sliding_attention",
|
| 105 |
+
"full_attention"
|
| 106 |
+
],
|
| 107 |
+
"max_position_embeddings": 262144,
|
| 108 |
+
"model_type": "gemma4_unified_text",
|
| 109 |
+
"moe_intermediate_size": null,
|
| 110 |
+
"num_attention_heads": 16,
|
| 111 |
+
"num_experts": null,
|
| 112 |
+
"num_global_key_value_heads": 1,
|
| 113 |
+
"num_hidden_layers": 48,
|
| 114 |
+
"num_key_value_heads": 8,
|
| 115 |
+
"num_kv_shared_layers": 0,
|
| 116 |
+
"pad_token_id": 0,
|
| 117 |
+
"rms_norm_eps": 1e-06,
|
| 118 |
+
"rope_parameters": {
|
| 119 |
+
"full_attention": {
|
| 120 |
+
"partial_rotary_factor": 0.25,
|
| 121 |
+
"rope_theta": 1000000.0,
|
| 122 |
+
"rope_type": "proportional"
|
| 123 |
+
},
|
| 124 |
+
"sliding_attention": {
|
| 125 |
+
"rope_theta": 10000.0,
|
| 126 |
+
"rope_type": "default"
|
| 127 |
+
}
|
| 128 |
+
},
|
| 129 |
+
"sliding_window": 1024,
|
| 130 |
+
"tie_word_embeddings": true,
|
| 131 |
+
"top_k_experts": null,
|
| 132 |
+
"use_bidirectional_attention": "vision",
|
| 133 |
+
"use_cache": false,
|
| 134 |
+
"use_double_wide_mlp": false,
|
| 135 |
+
"vocab_size": 262144,
|
| 136 |
+
"vocab_size_per_layer_input": 262144
|
| 137 |
+
},
|
| 138 |
+
"tie_word_embeddings": true,
|
| 139 |
+
"transformers_version": "5.10.0",
|
| 140 |
+
"use_cache": false,
|
| 141 |
+
"video_token_id": 258884,
|
| 142 |
+
"vision_config": {
|
| 143 |
+
"_name_or_path": "",
|
| 144 |
+
"architectures": null,
|
| 145 |
+
"chunk_size_feed_forward": 0,
|
| 146 |
+
"dtype": "bfloat16",
|
| 147 |
+
"id2label": {
|
| 148 |
+
"0": "LABEL_0",
|
| 149 |
+
"1": "LABEL_1"
|
| 150 |
+
},
|
| 151 |
+
"initializer_range": 0.02,
|
| 152 |
+
"is_encoder_decoder": false,
|
| 153 |
+
"label2id": {
|
| 154 |
+
"LABEL_0": 0,
|
| 155 |
+
"LABEL_1": 1
|
| 156 |
+
},
|
| 157 |
+
"mm_embed_dim": 3840,
|
| 158 |
+
"mm_posemb_size": 1120,
|
| 159 |
+
"model_type": "gemma4_unified_vision",
|
| 160 |
+
"num_soft_tokens": 280,
|
| 161 |
+
"output_attentions": false,
|
| 162 |
+
"output_hidden_states": false,
|
| 163 |
+
"output_proj_dims": 3840,
|
| 164 |
+
"patch_size": 16,
|
| 165 |
+
"pooling_kernel_size": 3,
|
| 166 |
+
"problem_type": null,
|
| 167 |
+
"return_dict": true,
|
| 168 |
+
"rms_norm_eps": 1e-06
|
| 169 |
+
}
|
| 170 |
+
}
|
g1_cleanup_code/1000M/generation_config.json
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token_id": 2,
|
| 3 |
+
"do_sample": true,
|
| 4 |
+
"eos_token_id": [
|
| 5 |
+
1,
|
| 6 |
+
1
|
| 7 |
+
],
|
| 8 |
+
"pad_token_id": 0,
|
| 9 |
+
"suppress_tokens": [
|
| 10 |
+
258883,
|
| 11 |
+
258882
|
| 12 |
+
],
|
| 13 |
+
"temperature": 1.0,
|
| 14 |
+
"top_k": 64,
|
| 15 |
+
"top_p": 0.95,
|
| 16 |
+
"transformers_version": "5.10.0"
|
| 17 |
+
}
|
g1_cleanup_code/1000M/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b8f08af2f18a9c2cd0de041f5e756e6732c4971e3740b41dc2587cbb09a958e3
|
| 3 |
+
size 23919549408
|
g1_cleanup_code/1000M/processor_config.json
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"audio_ms_per_token": 40,
|
| 3 |
+
"audio_seq_length": 750,
|
| 4 |
+
"feature_extractor": {
|
| 5 |
+
"audio_samples_per_token": 640,
|
| 6 |
+
"feature_extractor_type": "Gemma4UnifiedAudioFeatureExtractor",
|
| 7 |
+
"feature_size": 640,
|
| 8 |
+
"padding_side": "right",
|
| 9 |
+
"padding_value": 0.0,
|
| 10 |
+
"return_attention_mask": true,
|
| 11 |
+
"sampling_rate": 16000
|
| 12 |
+
},
|
| 13 |
+
"image_processor": {
|
| 14 |
+
"do_convert_rgb": true,
|
| 15 |
+
"do_normalize": false,
|
| 16 |
+
"do_rescale": true,
|
| 17 |
+
"do_resize": true,
|
| 18 |
+
"image_mean": [
|
| 19 |
+
0.0,
|
| 20 |
+
0.0,
|
| 21 |
+
0.0
|
| 22 |
+
],
|
| 23 |
+
"image_processor_type": "Gemma4UnifiedImageProcessor",
|
| 24 |
+
"image_std": [
|
| 25 |
+
1.0,
|
| 26 |
+
1.0,
|
| 27 |
+
1.0
|
| 28 |
+
],
|
| 29 |
+
"max_soft_tokens": 280,
|
| 30 |
+
"patch_size": 16,
|
| 31 |
+
"pooling_kernel_size": 3,
|
| 32 |
+
"resample": 3,
|
| 33 |
+
"rescale_factor": 0.00392156862745098
|
| 34 |
+
},
|
| 35 |
+
"image_seq_length": 280,
|
| 36 |
+
"processor_class": "Gemma4UnifiedProcessor",
|
| 37 |
+
"video_processor": {
|
| 38 |
+
"do_convert_rgb": true,
|
| 39 |
+
"do_normalize": true,
|
| 40 |
+
"do_rescale": true,
|
| 41 |
+
"do_resize": true,
|
| 42 |
+
"do_sample_frames": true,
|
| 43 |
+
"image_mean": [
|
| 44 |
+
0.0,
|
| 45 |
+
0.0,
|
| 46 |
+
0.0
|
| 47 |
+
],
|
| 48 |
+
"image_std": [
|
| 49 |
+
1.0,
|
| 50 |
+
1.0,
|
| 51 |
+
1.0
|
| 52 |
+
],
|
| 53 |
+
"max_soft_tokens": 70,
|
| 54 |
+
"num_frames": 32,
|
| 55 |
+
"patch_size": 16,
|
| 56 |
+
"pooling_kernel_size": 3,
|
| 57 |
+
"resample": 3,
|
| 58 |
+
"rescale_factor": 0.00392156862745098,
|
| 59 |
+
"return_metadata": false,
|
| 60 |
+
"video_processor_type": "Gemma4UnifiedVideoProcessor"
|
| 61 |
+
}
|
| 62 |
+
}
|
g1_cleanup_code/1000M/tokenizer.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:12bac982b793c44b03d52a250a9f0d0b666813da566b910c24a6da0695fd11e6
|
| 3 |
+
size 32170070
|
g1_cleanup_code/1000M/tokenizer_config.json
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"audio_token": "<|audio|>",
|
| 3 |
+
"backend": "tokenizers",
|
| 4 |
+
"boa_token": "<|audio>",
|
| 5 |
+
"boi_token": "<|image>",
|
| 6 |
+
"bos_token": "<bos>",
|
| 7 |
+
"eoa_token": "<audio|>",
|
| 8 |
+
"eoc_token": "<channel|>",
|
| 9 |
+
"eoi_token": "<image|>",
|
| 10 |
+
"eos_token": "<eos>",
|
| 11 |
+
"eot_token": "<turn|>",
|
| 12 |
+
"escape_token": "<|\"|>",
|
| 13 |
+
"etc_token": "<tool_call|>",
|
| 14 |
+
"etd_token": "<tool|>",
|
| 15 |
+
"etr_token": "<tool_response|>",
|
| 16 |
+
"extra_special_tokens": [
|
| 17 |
+
"<|video|>"
|
| 18 |
+
],
|
| 19 |
+
"image_token": "<|image|>",
|
| 20 |
+
"is_local": true,
|
| 21 |
+
"local_files_only": true,
|
| 22 |
+
"mask_token": "<mask>",
|
| 23 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 24 |
+
"model_specific_special_tokens": {
|
| 25 |
+
"audio_token": "<|audio|>",
|
| 26 |
+
"boa_token": "<|audio>",
|
| 27 |
+
"boi_token": "<|image>",
|
| 28 |
+
"eoa_token": "<audio|>",
|
| 29 |
+
"eoc_token": "<channel|>",
|
| 30 |
+
"eoi_token": "<image|>",
|
| 31 |
+
"eot_token": "<turn|>",
|
| 32 |
+
"escape_token": "<|\"|>",
|
| 33 |
+
"etc_token": "<tool_call|>",
|
| 34 |
+
"etd_token": "<tool|>",
|
| 35 |
+
"etr_token": "<tool_response|>",
|
| 36 |
+
"image_token": "<|image|>",
|
| 37 |
+
"soc_token": "<|channel>",
|
| 38 |
+
"sot_token": "<|turn>",
|
| 39 |
+
"stc_token": "<|tool_call>",
|
| 40 |
+
"std_token": "<|tool>",
|
| 41 |
+
"str_token": "<|tool_response>",
|
| 42 |
+
"think_token": "<|think|>"
|
| 43 |
+
},
|
| 44 |
+
"pad_token": "<pad>",
|
| 45 |
+
"padding_side": "right",
|
| 46 |
+
"processor_class": "Gemma4UnifiedProcessor",
|
| 47 |
+
"soc_token": "<|channel>",
|
| 48 |
+
"sot_token": "<|turn>",
|
| 49 |
+
"split_special_tokens": false,
|
| 50 |
+
"stc_token": "<|tool_call>",
|
| 51 |
+
"std_token": "<|tool>",
|
| 52 |
+
"str_token": "<|tool_response>",
|
| 53 |
+
"think_token": "<|think|>",
|
| 54 |
+
"tokenizer_class": "GemmaTokenizer",
|
| 55 |
+
"unk_token": "<unk>"
|
| 56 |
+
}
|
g1_cleanup_code/1000M/train_results.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"epoch": 0.7863013698630137,
|
| 3 |
+
"total_flos": 5300110976090112.0,
|
| 4 |
+
"train_loss": 0.05387794186535783,
|
| 5 |
+
"train_runtime": 14746.3499,
|
| 6 |
+
"train_samples_per_second": 9.108,
|
| 7 |
+
"train_steps_per_second": 0.759
|
| 8 |
+
}
|
g1_cleanup_code/1000M/trainer_log.jsonl
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"current_steps": 10180, "total_steps": 11193, "loss": 0.5852091312408447, "lr": 1.999957274187931e-05, "epoch": 0.7151387425360028, "percentage": 90.95, "elapsed_time": "0:01:01", "remaining_time": "0:00:06"}
|
| 2 |
+
{"current_steps": 10190, "total_steps": 11193, "loss": 0.624833059310913, "lr": 1.9991978062682906e-05, "epoch": 0.7158412363891816, "percentage": 91.04, "elapsed_time": "0:03:25", "remaining_time": "0:00:20"}
|
| 3 |
+
{"current_steps": 10200, "total_steps": 11193, "loss": 0.5865033626556396, "lr": 1.997489709987378e-05, "epoch": 0.7165437302423604, "percentage": 91.13, "elapsed_time": "0:05:49", "remaining_time": "0:00:34"}
|
| 4 |
+
{"current_steps": 10210, "total_steps": 11193, "loss": 0.6209458351135254, "lr": 1.9948346151502135e-05, "epoch": 0.7172462240955392, "percentage": 91.22, "elapsed_time": "0:08:14", "remaining_time": "0:00:47"}
|
| 5 |
+
{"current_steps": 10220, "total_steps": 11193, "loss": 0.5970417022705078, "lr": 1.991235055154325e-05, "epoch": 0.717948717948718, "percentage": 91.31, "elapsed_time": "0:10:38", "remaining_time": "0:01:00"}
|
| 6 |
+
{"current_steps": 10230, "total_steps": 11193, "loss": 0.6147752285003663, "lr": 1.9866944645724693e-05, "epoch": 0.7186512118018967, "percentage": 91.4, "elapsed_time": "0:13:02", "remaining_time": "0:01:13"}
|
| 7 |
+
{"current_steps": 10240, "total_steps": 11193, "loss": 0.6067847728729248, "lr": 1.9812171758754842e-05, "epoch": 0.7193537056550755, "percentage": 91.49, "elapsed_time": "0:15:26", "remaining_time": "0:01:26"}
|
| 8 |
+
{"current_steps": 10250, "total_steps": 11193, "loss": 0.6248685359954834, "lr": 1.9748084152983987e-05, "epoch": 0.7200561995082543, "percentage": 91.58, "elapsed_time": "0:17:52", "remaining_time": "0:01:38"}
|
| 9 |
+
{"current_steps": 10260, "total_steps": 11193, "loss": 0.6045065879821777, "lr": 1.967474297853744e-05, "epoch": 0.7207586933614331, "percentage": 91.66, "elapsed_time": "0:20:16", "remaining_time": "0:01:50"}
|
| 10 |
+
{"current_steps": 10270, "total_steps": 11193, "loss": 0.6001498222351074, "lr": 1.9592218214968256e-05, "epoch": 0.7214611872146118, "percentage": 91.75, "elapsed_time": "0:22:41", "remaining_time": "0:02:02"}
|
| 11 |
+
{"current_steps": 10280, "total_steps": 11193, "loss": 0.6187323570251465, "lr": 1.9500588604485213e-05, "epoch": 0.7221636810677906, "percentage": 91.84, "elapsed_time": "0:25:05", "remaining_time": "0:02:13"}
|
| 12 |
+
{"current_steps": 10290, "total_steps": 11193, "loss": 0.626680326461792, "lr": 1.939994157681979e-05, "epoch": 0.7228661749209695, "percentage": 91.93, "elapsed_time": "0:27:29", "remaining_time": "0:02:24"}
|
| 13 |
+
{"current_steps": 10300, "total_steps": 11193, "loss": 0.6163479804992675, "lr": 1.9290373165803803e-05, "epoch": 0.7235686687741483, "percentage": 92.02, "elapsed_time": "0:29:54", "remaining_time": "0:02:35"}
|
| 14 |
+
{"current_steps": 10310, "total_steps": 11193, "loss": 0.6144249916076661, "lr": 1.9171987917737328e-05, "epoch": 0.724271162627327, "percentage": 92.11, "elapsed_time": "0:32:18", "remaining_time": "0:02:46"}
|
| 15 |
+
{"current_steps": 10320, "total_steps": 11193, "loss": 0.6066095352172851, "lr": 1.9044898791634322e-05, "epoch": 0.7249736564805058, "percentage": 92.2, "elapsed_time": "0:34:43", "remaining_time": "0:02:56"}
|
| 16 |
+
{"current_steps": 10330, "total_steps": 11193, "loss": 0.6249143123626709, "lr": 1.8909227051441126e-05, "epoch": 0.7256761503336846, "percentage": 92.29, "elapsed_time": "0:37:07", "remaining_time": "0:03:06"}
|
| 17 |
+
{"current_steps": 10340, "total_steps": 11193, "loss": 0.6459973335266114, "lr": 1.8765102150330692e-05, "epoch": 0.7263786441868634, "percentage": 92.38, "elapsed_time": "0:39:32", "remaining_time": "0:03:15"}
|
| 18 |
+
{"current_steps": 10350, "total_steps": 11193, "loss": 0.6111915588378907, "lr": 1.861266160718297e-05, "epoch": 0.7270811380400422, "percentage": 92.47, "elapsed_time": "0:41:56", "remaining_time": "0:03:24"}
|
| 19 |
+
{"current_steps": 10360, "total_steps": 11193, "loss": 0.6064771175384521, "lr": 1.845205087536923e-05, "epoch": 0.7277836318932209, "percentage": 92.56, "elapsed_time": "0:44:20", "remaining_time": "0:03:33"}
|
| 20 |
+
{"current_steps": 10370, "total_steps": 11193, "loss": 0.6087264060974121, "lr": 1.8283423203965633e-05, "epoch": 0.7284861257463997, "percentage": 92.65, "elapsed_time": "0:46:44", "remaining_time": "0:03:42"}
|
| 21 |
+
{"current_steps": 10380, "total_steps": 11193, "loss": 0.598025131225586, "lr": 1.810693949152835e-05, "epoch": 0.7291886195995785, "percentage": 92.74, "elapsed_time": "0:49:09", "remaining_time": "0:03:50"}
|
| 22 |
+
{"current_steps": 10390, "total_steps": 11193, "loss": 0.5803035736083985, "lr": 1.7922768132569902e-05, "epoch": 0.7298911134527573, "percentage": 92.83, "elapsed_time": "0:51:34", "remaining_time": "0:03:59"}
|
| 23 |
+
{"current_steps": 10400, "total_steps": 11193, "loss": 0.6091340541839599, "lr": 1.773108485688302e-05, "epoch": 0.730593607305936, "percentage": 92.92, "elapsed_time": "0:53:58", "remaining_time": "0:04:06"}
|
| 24 |
+
{"current_steps": 10410, "total_steps": 11193, "loss": 0.6046764373779296, "lr": 1.7532072561865537e-05, "epoch": 0.7312961011591148, "percentage": 93.0, "elapsed_time": "0:56:23", "remaining_time": "0:04:14"}
|
| 25 |
+
{"current_steps": 10420, "total_steps": 11193, "loss": 0.60126953125, "lr": 1.732592113800616e-05, "epoch": 0.7319985950122937, "percentage": 93.09, "elapsed_time": "0:58:47", "remaining_time": "0:04:21"}
|
| 26 |
+
{"current_steps": 10430, "total_steps": 11193, "loss": 0.6024490356445312, "lr": 1.7112827287697668e-05, "epoch": 0.7327010888654725, "percentage": 93.18, "elapsed_time": "1:01:12", "remaining_time": "0:04:28"}
|
| 27 |
+
{"current_steps": 10440, "total_steps": 11193, "loss": 0.6132241249084472, "lr": 1.6892994337550452e-05, "epoch": 0.7334035827186512, "percentage": 93.27, "elapsed_time": "1:03:36", "remaining_time": "0:04:35"}
|
| 28 |
+
{"current_steps": 10450, "total_steps": 11193, "loss": 0.6058990478515625, "lr": 1.6666632044385487e-05, "epoch": 0.73410607657183, "percentage": 93.36, "elapsed_time": "1:06:01", "remaining_time": "0:04:41"}
|
| 29 |
+
{"current_steps": 10460, "total_steps": 11193, "loss": 0.6276742458343506, "lr": 1.6433956395091757e-05, "epoch": 0.7348085704250088, "percentage": 93.45, "elapsed_time": "1:08:26", "remaining_time": "0:04:47"}
|
| 30 |
+
{"current_steps": 10470, "total_steps": 11193, "loss": 0.6159543037414551, "lr": 1.6195189400539238e-05, "epoch": 0.7355110642781876, "percentage": 93.54, "elapsed_time": "1:10:50", "remaining_time": "0:04:53"}
|
| 31 |
+
{"current_steps": 10480, "total_steps": 11193, "loss": 0.5714602470397949, "lr": 1.595055888374398e-05, "epoch": 0.7362135581313664, "percentage": 93.63, "elapsed_time": "1:13:15", "remaining_time": "0:04:59"}
|
| 32 |
+
{"current_steps": 10490, "total_steps": 11193, "loss": 0.6032049179077148, "lr": 1.5700298262487434e-05, "epoch": 0.7369160519845451, "percentage": 93.72, "elapsed_time": "1:15:39", "remaining_time": "0:05:04"}
|
| 33 |
+
{"current_steps": 10500, "total_steps": 11193, "loss": 0.5805444717407227, "lr": 1.5444646326597467e-05, "epoch": 0.7376185458377239, "percentage": 93.81, "elapsed_time": "1:18:03", "remaining_time": "0:05:09"}
|
| 34 |
+
{"current_steps": 10510, "total_steps": 11193, "loss": 0.5913052558898926, "lr": 1.5183847010103586e-05, "epoch": 0.7383210396909027, "percentage": 93.9, "elapsed_time": "1:20:28", "remaining_time": "0:05:13"}
|
| 35 |
+
{"current_steps": 10520, "total_steps": 11193, "loss": 0.5918679714202881, "lr": 1.4918149158483686e-05, "epoch": 0.7390235335440815, "percentage": 93.99, "elapsed_time": "1:22:52", "remaining_time": "0:05:18"}
|
| 36 |
+
{"current_steps": 10530, "total_steps": 11193, "loss": 0.6032553672790527, "lr": 1.464780629122452e-05, "epoch": 0.7397260273972602, "percentage": 94.08, "elapsed_time": "1:25:18", "remaining_time": "0:05:22"}
|
| 37 |
+
{"current_steps": 10540, "total_steps": 11193, "loss": 0.5595372676849365, "lr": 1.4373076359922374e-05, "epoch": 0.740428521250439, "percentage": 94.17, "elapsed_time": "1:27:43", "remaining_time": "0:05:26"}
|
| 38 |
+
{"current_steps": 10550, "total_steps": 11193, "loss": 0.6196616649627685, "lr": 1.4094221502154774e-05, "epoch": 0.7411310151036179, "percentage": 94.26, "elapsed_time": "1:30:07", "remaining_time": "0:05:29"}
|
| 39 |
+
{"current_steps": 10560, "total_steps": 11193, "loss": 0.6003097057342529, "lr": 1.3811507791358113e-05, "epoch": 0.7418335089567967, "percentage": 94.34, "elapsed_time": "1:32:31", "remaining_time": "0:05:32"}
|
| 40 |
+
{"current_steps": 10570, "total_steps": 11193, "loss": 0.6162596702575683, "lr": 1.3525204982949753e-05, "epoch": 0.7425360028099754, "percentage": 94.43, "elapsed_time": "1:34:56", "remaining_time": "0:05:35"}
|
| 41 |
+
{"current_steps": 10580, "total_steps": 11193, "loss": 0.6030051708221436, "lr": 1.3235586256936996e-05, "epoch": 0.7432384966631542, "percentage": 94.52, "elapsed_time": "1:37:20", "remaining_time": "0:05:38"}
|
| 42 |
+
{"current_steps": 10590, "total_steps": 11193, "loss": 0.619550085067749, "lr": 1.294292795725836e-05, "epoch": 0.743940990516333, "percentage": 94.61, "elapsed_time": "1:39:44", "remaining_time": "0:05:40"}
|
| 43 |
+
{"current_steps": 10600, "total_steps": 11193, "loss": 0.591694974899292, "lr": 1.2647509328105978e-05, "epoch": 0.7446434843695118, "percentage": 94.7, "elapsed_time": "1:42:10", "remaining_time": "0:05:42"}
|
| 44 |
+
{"current_steps": 10610, "total_steps": 11193, "loss": 0.5779125213623046, "lr": 1.2349612247480686e-05, "epoch": 0.7453459782226906, "percentage": 94.79, "elapsed_time": "1:44:34", "remaining_time": "0:05:44"}
|
| 45 |
+
{"current_steps": 10620, "total_steps": 11193, "loss": 0.6079377174377442, "lr": 1.2049520958234e-05, "epoch": 0.7460484720758693, "percentage": 94.88, "elapsed_time": "1:46:58", "remaining_time": "0:05:46"}
|
| 46 |
+
{"current_steps": 10630, "total_steps": 11193, "loss": 0.5842973232269287, "lr": 1.1747521796853638e-05, "epoch": 0.7467509659290481, "percentage": 94.97, "elapsed_time": "1:49:22", "remaining_time": "0:05:47"}
|
| 47 |
+
{"current_steps": 10640, "total_steps": 11193, "loss": 0.611078405380249, "lr": 1.1443902920251395e-05, "epoch": 0.7474534597822269, "percentage": 95.06, "elapsed_time": "1:51:46", "remaining_time": "0:05:48"}
|
| 48 |
+
{"current_steps": 10650, "total_steps": 11193, "loss": 0.5831323146820069, "lr": 1.1138954030814009e-05, "epoch": 0.7481559536354057, "percentage": 95.15, "elapsed_time": "1:54:11", "remaining_time": "0:05:49"}
|
| 49 |
+
{"current_steps": 10660, "total_steps": 11193, "loss": 0.5817962646484375, "lr": 1.0832966099979388e-05, "epoch": 0.7488584474885844, "percentage": 95.24, "elapsed_time": "1:56:47", "remaining_time": "0:05:50"}
|
| 50 |
+
{"current_steps": 10670, "total_steps": 11193, "loss": 0.587282133102417, "lr": 1.0526231090601969e-05, "epoch": 0.7495609413417632, "percentage": 95.33, "elapsed_time": "1:59:11", "remaining_time": "0:05:50"}
|
| 51 |
+
{"current_steps": 10680, "total_steps": 11193, "loss": 0.5874773979187011, "lr": 1.0219041678372106e-05, "epoch": 0.7502634351949421, "percentage": 95.42, "elapsed_time": "2:01:36", "remaining_time": "0:05:50"}
|
| 52 |
+
{"current_steps": 10690, "total_steps": 11193, "loss": 0.5797173976898193, "lr": 9.911690972555268e-06, "epoch": 0.7509659290481209, "percentage": 95.51, "elapsed_time": "2:04:00", "remaining_time": "0:05:50"}
|
| 53 |
+
{"current_steps": 10700, "total_steps": 11193, "loss": 0.6123420238494873, "lr": 9.604472236317541e-06, "epoch": 0.7516684229012996, "percentage": 95.6, "elapsed_time": "2:06:24", "remaining_time": "0:05:49"}
|
| 54 |
+
{"current_steps": 10710, "total_steps": 11193, "loss": 0.5702229976654053, "lr": 9.297678606904335e-06, "epoch": 0.7523709167544784, "percentage": 95.68, "elapsed_time": "2:08:49", "remaining_time": "0:05:48"}
|
| 55 |
+
{"current_steps": 10720, "total_steps": 11193, "loss": 0.5769252777099609, "lr": 8.991602815939155e-06, "epoch": 0.7530734106076572, "percentage": 95.77, "elapsed_time": "2:11:13", "remaining_time": "0:05:47"}
|
| 56 |
+
{"current_steps": 10730, "total_steps": 11193, "loss": 0.5780287742614746, "lr": 8.686536910109464e-06, "epoch": 0.753775904460836, "percentage": 95.86, "elapsed_time": "2:13:37", "remaining_time": "0:05:45"}
|
| 57 |
+
{"current_steps": 10740, "total_steps": 11193, "loss": 0.6100452423095704, "lr": 8.382771972506084e-06, "epoch": 0.7544783983140148, "percentage": 95.95, "elapsed_time": "2:16:02", "remaining_time": "0:05:44"}
|
| 58 |
+
{"current_steps": 10750, "total_steps": 11193, "loss": 0.5884391784667968, "lr": 8.08059784488198e-06, "epoch": 0.7551808921671935, "percentage": 96.04, "elapsed_time": "2:18:27", "remaining_time": "0:05:42"}
|
| 59 |
+
{"current_steps": 10760, "total_steps": 11193, "loss": 0.5781902313232422, "lr": 7.780302851095544e-06, "epoch": 0.7558833860203723, "percentage": 96.13, "elapsed_time": "2:20:52", "remaining_time": "0:05:40"}
|
| 60 |
+
{"current_steps": 10770, "total_steps": 11193, "loss": 0.5921500205993653, "lr": 7.482173522002166e-06, "epoch": 0.7565858798735511, "percentage": 96.22, "elapsed_time": "2:23:16", "remaining_time": "0:05:37"}
|
| 61 |
+
{"current_steps": 10780, "total_steps": 11193, "loss": 0.6058819770812989, "lr": 7.186494322056624e-06, "epoch": 0.7572883737267299, "percentage": 96.31, "elapsed_time": "2:25:40", "remaining_time": "0:05:34"}
|
| 62 |
+
{"current_steps": 10790, "total_steps": 11193, "loss": 0.6233699798583985, "lr": 6.893547377887183e-06, "epoch": 0.7579908675799086, "percentage": 96.4, "elapsed_time": "2:28:04", "remaining_time": "0:05:31"}
|
| 63 |
+
{"current_steps": 10800, "total_steps": 11193, "loss": 0.5790233612060547, "lr": 6.6036122091003365e-06, "epoch": 0.7586933614330874, "percentage": 96.49, "elapsed_time": "2:30:29", "remaining_time": "0:05:28"}
|
| 64 |
+
{"current_steps": 10810, "total_steps": 11193, "loss": 0.5863173484802247, "lr": 6.316965461573097e-06, "epoch": 0.7593958552862663, "percentage": 96.58, "elapsed_time": "2:32:54", "remaining_time": "0:05:25"}
|
| 65 |
+
{"current_steps": 10820, "total_steps": 11193, "loss": 0.592318344116211, "lr": 6.0338806434872965e-06, "epoch": 0.7600983491394451, "percentage": 96.67, "elapsed_time": "2:35:39", "remaining_time": "0:05:21"}
|
| 66 |
+
{"current_steps": 10830, "total_steps": 11193, "loss": 0.5882054805755615, "lr": 5.754627864357749e-06, "epoch": 0.7608008429926238, "percentage": 96.76, "elapsed_time": "2:38:03", "remaining_time": "0:05:17"}
|
| 67 |
+
{"current_steps": 10840, "total_steps": 11193, "loss": 0.5774524688720704, "lr": 5.479473577303319e-06, "epoch": 0.7615033368458026, "percentage": 96.85, "elapsed_time": "2:40:28", "remaining_time": "0:05:13"}
|
| 68 |
+
{"current_steps": 10850, "total_steps": 11193, "loss": 0.5922804832458496, "lr": 5.208680324806778e-06, "epoch": 0.7622058306989814, "percentage": 96.94, "elapsed_time": "2:42:52", "remaining_time": "0:05:08"}
|
| 69 |
+
{"current_steps": 10860, "total_steps": 11193, "loss": 0.570035171508789, "lr": 4.94250648820606e-06, "epoch": 0.7629083245521602, "percentage": 97.02, "elapsed_time": "2:45:16", "remaining_time": "0:05:04"}
|
| 70 |
+
{"current_steps": 10870, "total_steps": 11193, "loss": 0.5976253509521484, "lr": 4.681206041155951e-06, "epoch": 0.763610818405339, "percentage": 97.11, "elapsed_time": "2:47:40", "remaining_time": "0:04:58"}
|
| 71 |
+
{"current_steps": 10880, "total_steps": 11193, "loss": 0.5517823219299316, "lr": 4.42502830729538e-06, "epoch": 0.7643133122585177, "percentage": 97.2, "elapsed_time": "2:50:05", "remaining_time": "0:04:53"}
|
| 72 |
+
{"current_steps": 10890, "total_steps": 11193, "loss": 0.6046184539794922, "lr": 4.174217722351669e-06, "epoch": 0.7650158061116965, "percentage": 97.29, "elapsed_time": "2:52:30", "remaining_time": "0:04:47"}
|
| 73 |
+
{"current_steps": 10900, "total_steps": 11193, "loss": 0.5755278587341308, "lr": 3.929013600908575e-06, "epoch": 0.7657182999648753, "percentage": 97.38, "elapsed_time": "2:54:54", "remaining_time": "0:04:42"}
|
| 74 |
+
{"current_steps": 10910, "total_steps": 11193, "loss": 0.5779466152191162, "lr": 3.689649908060807e-06, "epoch": 0.7664207938180541, "percentage": 97.47, "elapsed_time": "2:57:18", "remaining_time": "0:04:35"}
|
| 75 |
+
{"current_steps": 10920, "total_steps": 11193, "loss": 0.5788602352142334, "lr": 3.4563550361728028e-06, "epoch": 0.7671232876712328, "percentage": 97.56, "elapsed_time": "2:59:43", "remaining_time": "0:04:29"}
|
| 76 |
+
{"current_steps": 10930, "total_steps": 11193, "loss": 0.5751082420349121, "lr": 3.2293515869548216e-06, "epoch": 0.7678257815244116, "percentage": 97.65, "elapsed_time": "3:02:07", "remaining_time": "0:04:22"}
|
| 77 |
+
{"current_steps": 10940, "total_steps": 11193, "loss": 0.5937278270721436, "lr": 3.0088561590642746e-06, "epoch": 0.7685282753775905, "percentage": 97.74, "elapsed_time": "3:04:31", "remaining_time": "0:04:16"}
|
| 78 |
+
{"current_steps": 10950, "total_steps": 11193, "loss": 0.5773450374603272, "lr": 2.795079141434941e-06, "epoch": 0.7692307692307693, "percentage": 97.83, "elapsed_time": "3:06:56", "remaining_time": "0:04:08"}
|
| 79 |
+
{"current_steps": 10960, "total_steps": 11193, "loss": 0.5806230545043946, "lr": 2.588224512531299e-06, "epoch": 0.769933263083948, "percentage": 97.92, "elapsed_time": "3:09:21", "remaining_time": "0:04:01"}
|
| 80 |
+
{"current_steps": 10970, "total_steps": 11193, "loss": 0.5859660625457763, "lr": 2.388489645719495e-06, "epoch": 0.7706357569371268, "percentage": 98.01, "elapsed_time": "3:11:45", "remaining_time": "0:03:53"}
|
| 81 |
+
{"current_steps": 10980, "total_steps": 11193, "loss": 0.5792638301849365, "lr": 2.196065120940663e-06, "epoch": 0.7713382507903056, "percentage": 98.1, "elapsed_time": "3:14:09", "remaining_time": "0:03:45"}
|
| 82 |
+
{"current_steps": 10990, "total_steps": 11193, "loss": 0.5705962181091309, "lr": 2.0111345428662808e-06, "epoch": 0.7720407446434844, "percentage": 98.19, "elapsed_time": "3:16:33", "remaining_time": "0:03:37"}
|
| 83 |
+
{"current_steps": 11000, "total_steps": 11193, "loss": 0.5686861991882324, "lr": 1.8338743657091102e-06, "epoch": 0.7727432384966632, "percentage": 98.28, "elapsed_time": "3:18:58", "remaining_time": "0:03:29"}
|
| 84 |
+
{"current_steps": 11010, "total_steps": 11193, "loss": 0.5631584167480469, "lr": 1.6644537248568113e-06, "epoch": 0.7734457323498419, "percentage": 98.37, "elapsed_time": "3:21:22", "remaining_time": "0:03:20"}
|
| 85 |
+
{"current_steps": 11020, "total_steps": 11193, "loss": 0.5635076045989991, "lr": 1.503034275488951e-06, "epoch": 0.7741482262030207, "percentage": 98.45, "elapsed_time": "3:23:46", "remaining_time": "0:03:11"}
|
| 86 |
+
{"current_steps": 11030, "total_steps": 11193, "loss": 0.5801867961883544, "lr": 1.3497700383313495e-06, "epoch": 0.7748507200561995, "percentage": 98.54, "elapsed_time": "3:26:11", "remaining_time": "0:03:02"}
|
| 87 |
+
{"current_steps": 11040, "total_steps": 11193, "loss": 0.5985492706298828, "lr": 1.2048072526949562e-06, "epoch": 0.7755532139093783, "percentage": 98.63, "elapsed_time": "3:28:35", "remaining_time": "0:02:53"}
|
| 88 |
+
{"current_steps": 11050, "total_steps": 11193, "loss": 0.5673690795898437, "lr": 1.0682842369394756e-06, "epoch": 0.776255707762557, "percentage": 98.72, "elapsed_time": "3:31:00", "remaining_time": "0:02:43"}
|
| 89 |
+
{"current_steps": 11060, "total_steps": 11193, "loss": 0.6037036895751953, "lr": 9.403312564948792e-07, "epoch": 0.7769582016157358, "percentage": 98.81, "elapsed_time": "3:33:24", "remaining_time": "0:02:33"}
|
| 90 |
+
{"current_steps": 11070, "total_steps": 11193, "loss": 0.5852627754211426, "lr": 8.210703995667471e-07, "epoch": 0.7776606954689147, "percentage": 98.9, "elapsed_time": "3:35:48", "remaining_time": "0:02:23"}
|
| 91 |
+
{"current_steps": 11080, "total_steps": 11193, "loss": 0.5962277412414551, "lr": 7.106154606440245e-07, "epoch": 0.7783631893220935, "percentage": 98.99, "elapsed_time": "3:38:12", "remaining_time": "0:02:13"}
|
| 92 |
+
{"current_steps": 11090, "total_steps": 11193, "loss": 0.5665205955505371, "lr": 6.090718319203393e-07, "epoch": 0.7790656831752722, "percentage": 99.08, "elapsed_time": "3:40:37", "remaining_time": "0:02:02"}
|
| 93 |
+
{"current_steps": 11100, "total_steps": 11193, "loss": 0.5832221984863282, "lr": 5.165364027325089e-07, "epoch": 0.779768177028451, "percentage": 99.17, "elapsed_time": "3:43:02", "remaining_time": "0:01:52"}
|
| 94 |
+
{"current_steps": 11110, "total_steps": 11193, "loss": 0.5794486999511719, "lr": 4.330974671121585e-07, "epoch": 0.7804706708816298, "percentage": 99.26, "elapsed_time": "3:45:26", "remaining_time": "0:01:41"}
|
| 95 |
+
{"current_steps": 11120, "total_steps": 11193, "loss": 0.5899890422821045, "lr": 3.588346395386846e-07, "epoch": 0.7811731647348086, "percentage": 99.35, "elapsed_time": "3:47:50", "remaining_time": "0:01:29"}
|
| 96 |
+
{"current_steps": 11130, "total_steps": 11193, "loss": 0.5896859645843506, "lr": 2.938187789739395e-07, "epoch": 0.7818756585879874, "percentage": 99.44, "elapsed_time": "3:50:15", "remaining_time": "0:01:18"}
|
| 97 |
+
{"current_steps": 11140, "total_steps": 11193, "loss": 0.5642678737640381, "lr": 2.3811192125110724e-07, "epoch": 0.7825781524411661, "percentage": 99.53, "elapsed_time": "3:52:39", "remaining_time": "0:01:06"}
|
| 98 |
+
{"current_steps": 11150, "total_steps": 11193, "loss": 0.5775588512420654, "lr": 1.917672198823177e-07, "epoch": 0.7832806462943449, "percentage": 99.62, "elapsed_time": "3:55:03", "remaining_time": "0:00:54"}
|
| 99 |
+
{"current_steps": 11160, "total_steps": 11193, "loss": 0.5753410339355469, "lr": 1.5482889534143795e-07, "epoch": 0.7839831401475237, "percentage": 99.71, "elapsed_time": "3:57:28", "remaining_time": "0:00:42"}
|
| 100 |
+
{"current_steps": 11170, "total_steps": 11193, "loss": 0.5725598335266113, "lr": 1.273321928704694e-07, "epoch": 0.7846856340007025, "percentage": 99.79, "elapsed_time": "3:59:53", "remaining_time": "0:00:29"}
|
| 101 |
+
{"current_steps": 11180, "total_steps": 11193, "loss": 0.5798064231872558, "lr": 1.0930334884977866e-07, "epoch": 0.7853881278538812, "percentage": 99.88, "elapsed_time": "4:02:17", "remaining_time": "0:00:16"}
|
| 102 |
+
{"current_steps": 11190, "total_steps": 11193, "loss": 0.5891654014587402, "lr": 1.0075956576427487e-07, "epoch": 0.78609062170706, "percentage": 99.97, "elapsed_time": "4:04:42", "remaining_time": "0:00:03"}
|
| 103 |
+
{"current_steps": 11193, "total_steps": 11193, "epoch": 0.7863013698630137, "percentage": 100.0, "elapsed_time": "4:05:46", "remaining_time": "0:00:00"}
|
g1_cleanup_code/1000M/trainer_state.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
g1_cleanup_code/1000M/training_args.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:24e8dd0aea22bac7539678a831f485e6dd20b1816e427788691df9d9de1c0ad9
|
| 3 |
+
size 8209
|
g1_cleanup_code/1000M/training_loss.png
ADDED
|