rqadri commited on
Commit
d84ed65
·
verified ·
1 Parent(s): 1c9273a

Upload folder using huggingface_hub

Browse files
.gitattributes CHANGED
@@ -47,3 +47,4 @@ g1_blog_post__edit/500M/tokenizer.json filter=lfs diff=lfs merge=lfs -text
47
  g1_blog_post__edit/1000M/tokenizer.json filter=lfs diff=lfs merge=lfs -text
48
  g1_cleanup_code/250M/tokenizer.json filter=lfs diff=lfs merge=lfs -text
49
  g1_cleanup_code/500M/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 
 
47
  g1_blog_post__edit/1000M/tokenizer.json filter=lfs diff=lfs merge=lfs -text
48
  g1_cleanup_code/250M/tokenizer.json filter=lfs diff=lfs merge=lfs -text
49
  g1_cleanup_code/500M/tokenizer.json filter=lfs diff=lfs merge=lfs -text
50
+ g1_cleanup_code/1000M/tokenizer.json filter=lfs diff=lfs merge=lfs -text
g1_cleanup_code/1000M/README.md ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ license: other
4
+ tags:
5
+ - llama-factory
6
+ - full
7
+ - generated_from_trainer
8
+ model-index:
9
+ - name: 1000M
10
+ results: []
11
+ ---
12
+
13
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
14
+ should probably proofread and complete it, then remove this comment. -->
15
+
16
+ # 1000M
17
+
18
+ This model is a fine-tuned version of [/lus/flare/projects/MultiModalLLM/model-weights/gemma_base](https://huggingface.co//lus/flare/projects/MultiModalLLM/model-weights/gemma_base) on the rv5_g1_cleanup_code and the instruct_replay_gemma datasets.
19
+
20
+ ## Model description
21
+
22
+ More information needed
23
+
24
+ ## Intended uses & limitations
25
+
26
+ More information needed
27
+
28
+ ## Training and evaluation data
29
+
30
+ More information needed
31
+
32
+ ## Training procedure
33
+
34
+ ### Training hyperparameters
35
+
36
+ The following hyperparameters were used during training:
37
+ - learning_rate: 2e-05
38
+ - train_batch_size: 1
39
+ - eval_batch_size: 8
40
+ - seed: 42
41
+ - distributed_type: multi-GPU
42
+ - num_devices: 12
43
+ - total_train_batch_size: 12
44
+ - total_eval_batch_size: 96
45
+ - optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.95) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
46
+ - lr_scheduler_type: warmup_stable_decay
47
+ - lr_scheduler_warmup_steps: 178
48
+ - training_steps: 11193
49
+
50
+ ### Training results
51
+
52
+
53
+
54
+ ### Framework versions
55
+
56
+ - Transformers 5.10.0
57
+ - Pytorch 2.10.0a0+git449b176
58
+ - Datasets 4.5.0
59
+ - Tokenizers 0.22.2
g1_cleanup_code/1000M/all_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 0.7863013698630137,
3
+ "total_flos": 5300110976090112.0,
4
+ "train_loss": 0.05387794186535783,
5
+ "train_runtime": 14746.3499,
6
+ "train_samples_per_second": 9.108,
7
+ "train_steps_per_second": 0.759
8
+ }
g1_cleanup_code/1000M/chat_template.jinja ADDED
@@ -0,0 +1 @@
 
 
1
+ {% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content + '<eos>' }}{% endif %}{% endfor %}
g1_cleanup_code/1000M/config.json ADDED
@@ -0,0 +1,170 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Gemma4UnifiedForConditionalGeneration"
4
+ ],
5
+ "audio_config": {
6
+ "_name_or_path": "",
7
+ "architectures": null,
8
+ "audio_embed_dim": 640,
9
+ "chunk_size_feed_forward": 0,
10
+ "dtype": "bfloat16",
11
+ "id2label": {
12
+ "0": "LABEL_0",
13
+ "1": "LABEL_1"
14
+ },
15
+ "initializer_range": 0.02,
16
+ "is_encoder_decoder": false,
17
+ "label2id": {
18
+ "LABEL_0": 0,
19
+ "LABEL_1": 1
20
+ },
21
+ "model_type": "gemma4_unified_audio",
22
+ "output_attentions": false,
23
+ "output_hidden_states": false,
24
+ "problem_type": null,
25
+ "return_dict": true,
26
+ "rms_norm_eps": 1e-06
27
+ },
28
+ "audio_token_id": 258881,
29
+ "boa_token_id": 256000,
30
+ "boi_token_id": 255999,
31
+ "bos_token_id": 2,
32
+ "dtype": "bfloat16",
33
+ "eoa_token_index": 258883,
34
+ "eoi_token_id": 258882,
35
+ "eos_token_id": 1,
36
+ "hidden_size": 3840,
37
+ "image_token_id": 258880,
38
+ "initializer_range": 0.02,
39
+ "model_type": "gemma4_unified",
40
+ "pad_token_id": 0,
41
+ "text_config": {
42
+ "attention_bias": false,
43
+ "attention_dropout": 0.0,
44
+ "attention_k_eq_v": true,
45
+ "bos_token_id": 2,
46
+ "dtype": "bfloat16",
47
+ "enable_moe_block": false,
48
+ "eos_token_id": 1,
49
+ "final_logit_softcapping": 30.0,
50
+ "global_head_dim": 512,
51
+ "head_dim": 256,
52
+ "hidden_activation": "gelu_pytorch_tanh",
53
+ "hidden_size": 3840,
54
+ "hidden_size_per_layer_input": 0,
55
+ "initializer_range": 0.02,
56
+ "intermediate_size": 15360,
57
+ "layer_types": [
58
+ "sliding_attention",
59
+ "sliding_attention",
60
+ "sliding_attention",
61
+ "sliding_attention",
62
+ "sliding_attention",
63
+ "full_attention",
64
+ "sliding_attention",
65
+ "sliding_attention",
66
+ "sliding_attention",
67
+ "sliding_attention",
68
+ "sliding_attention",
69
+ "full_attention",
70
+ "sliding_attention",
71
+ "sliding_attention",
72
+ "sliding_attention",
73
+ "sliding_attention",
74
+ "sliding_attention",
75
+ "full_attention",
76
+ "sliding_attention",
77
+ "sliding_attention",
78
+ "sliding_attention",
79
+ "sliding_attention",
80
+ "sliding_attention",
81
+ "full_attention",
82
+ "sliding_attention",
83
+ "sliding_attention",
84
+ "sliding_attention",
85
+ "sliding_attention",
86
+ "sliding_attention",
87
+ "full_attention",
88
+ "sliding_attention",
89
+ "sliding_attention",
90
+ "sliding_attention",
91
+ "sliding_attention",
92
+ "sliding_attention",
93
+ "full_attention",
94
+ "sliding_attention",
95
+ "sliding_attention",
96
+ "sliding_attention",
97
+ "sliding_attention",
98
+ "sliding_attention",
99
+ "full_attention",
100
+ "sliding_attention",
101
+ "sliding_attention",
102
+ "sliding_attention",
103
+ "sliding_attention",
104
+ "sliding_attention",
105
+ "full_attention"
106
+ ],
107
+ "max_position_embeddings": 262144,
108
+ "model_type": "gemma4_unified_text",
109
+ "moe_intermediate_size": null,
110
+ "num_attention_heads": 16,
111
+ "num_experts": null,
112
+ "num_global_key_value_heads": 1,
113
+ "num_hidden_layers": 48,
114
+ "num_key_value_heads": 8,
115
+ "num_kv_shared_layers": 0,
116
+ "pad_token_id": 0,
117
+ "rms_norm_eps": 1e-06,
118
+ "rope_parameters": {
119
+ "full_attention": {
120
+ "partial_rotary_factor": 0.25,
121
+ "rope_theta": 1000000.0,
122
+ "rope_type": "proportional"
123
+ },
124
+ "sliding_attention": {
125
+ "rope_theta": 10000.0,
126
+ "rope_type": "default"
127
+ }
128
+ },
129
+ "sliding_window": 1024,
130
+ "tie_word_embeddings": true,
131
+ "top_k_experts": null,
132
+ "use_bidirectional_attention": "vision",
133
+ "use_cache": false,
134
+ "use_double_wide_mlp": false,
135
+ "vocab_size": 262144,
136
+ "vocab_size_per_layer_input": 262144
137
+ },
138
+ "tie_word_embeddings": true,
139
+ "transformers_version": "5.10.0",
140
+ "use_cache": false,
141
+ "video_token_id": 258884,
142
+ "vision_config": {
143
+ "_name_or_path": "",
144
+ "architectures": null,
145
+ "chunk_size_feed_forward": 0,
146
+ "dtype": "bfloat16",
147
+ "id2label": {
148
+ "0": "LABEL_0",
149
+ "1": "LABEL_1"
150
+ },
151
+ "initializer_range": 0.02,
152
+ "is_encoder_decoder": false,
153
+ "label2id": {
154
+ "LABEL_0": 0,
155
+ "LABEL_1": 1
156
+ },
157
+ "mm_embed_dim": 3840,
158
+ "mm_posemb_size": 1120,
159
+ "model_type": "gemma4_unified_vision",
160
+ "num_soft_tokens": 280,
161
+ "output_attentions": false,
162
+ "output_hidden_states": false,
163
+ "output_proj_dims": 3840,
164
+ "patch_size": 16,
165
+ "pooling_kernel_size": 3,
166
+ "problem_type": null,
167
+ "return_dict": true,
168
+ "rms_norm_eps": 1e-06
169
+ }
170
+ }
g1_cleanup_code/1000M/generation_config.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 2,
3
+ "do_sample": true,
4
+ "eos_token_id": [
5
+ 1,
6
+ 1
7
+ ],
8
+ "pad_token_id": 0,
9
+ "suppress_tokens": [
10
+ 258883,
11
+ 258882
12
+ ],
13
+ "temperature": 1.0,
14
+ "top_k": 64,
15
+ "top_p": 0.95,
16
+ "transformers_version": "5.10.0"
17
+ }
g1_cleanup_code/1000M/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b8f08af2f18a9c2cd0de041f5e756e6732c4971e3740b41dc2587cbb09a958e3
3
+ size 23919549408
g1_cleanup_code/1000M/processor_config.json ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "audio_ms_per_token": 40,
3
+ "audio_seq_length": 750,
4
+ "feature_extractor": {
5
+ "audio_samples_per_token": 640,
6
+ "feature_extractor_type": "Gemma4UnifiedAudioFeatureExtractor",
7
+ "feature_size": 640,
8
+ "padding_side": "right",
9
+ "padding_value": 0.0,
10
+ "return_attention_mask": true,
11
+ "sampling_rate": 16000
12
+ },
13
+ "image_processor": {
14
+ "do_convert_rgb": true,
15
+ "do_normalize": false,
16
+ "do_rescale": true,
17
+ "do_resize": true,
18
+ "image_mean": [
19
+ 0.0,
20
+ 0.0,
21
+ 0.0
22
+ ],
23
+ "image_processor_type": "Gemma4UnifiedImageProcessor",
24
+ "image_std": [
25
+ 1.0,
26
+ 1.0,
27
+ 1.0
28
+ ],
29
+ "max_soft_tokens": 280,
30
+ "patch_size": 16,
31
+ "pooling_kernel_size": 3,
32
+ "resample": 3,
33
+ "rescale_factor": 0.00392156862745098
34
+ },
35
+ "image_seq_length": 280,
36
+ "processor_class": "Gemma4UnifiedProcessor",
37
+ "video_processor": {
38
+ "do_convert_rgb": true,
39
+ "do_normalize": true,
40
+ "do_rescale": true,
41
+ "do_resize": true,
42
+ "do_sample_frames": true,
43
+ "image_mean": [
44
+ 0.0,
45
+ 0.0,
46
+ 0.0
47
+ ],
48
+ "image_std": [
49
+ 1.0,
50
+ 1.0,
51
+ 1.0
52
+ ],
53
+ "max_soft_tokens": 70,
54
+ "num_frames": 32,
55
+ "patch_size": 16,
56
+ "pooling_kernel_size": 3,
57
+ "resample": 3,
58
+ "rescale_factor": 0.00392156862745098,
59
+ "return_metadata": false,
60
+ "video_processor_type": "Gemma4UnifiedVideoProcessor"
61
+ }
62
+ }
g1_cleanup_code/1000M/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:12bac982b793c44b03d52a250a9f0d0b666813da566b910c24a6da0695fd11e6
3
+ size 32170070
g1_cleanup_code/1000M/tokenizer_config.json ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "audio_token": "<|audio|>",
3
+ "backend": "tokenizers",
4
+ "boa_token": "<|audio>",
5
+ "boi_token": "<|image>",
6
+ "bos_token": "<bos>",
7
+ "eoa_token": "<audio|>",
8
+ "eoc_token": "<channel|>",
9
+ "eoi_token": "<image|>",
10
+ "eos_token": "<eos>",
11
+ "eot_token": "<turn|>",
12
+ "escape_token": "<|\"|>",
13
+ "etc_token": "<tool_call|>",
14
+ "etd_token": "<tool|>",
15
+ "etr_token": "<tool_response|>",
16
+ "extra_special_tokens": [
17
+ "<|video|>"
18
+ ],
19
+ "image_token": "<|image|>",
20
+ "is_local": true,
21
+ "local_files_only": true,
22
+ "mask_token": "<mask>",
23
+ "model_max_length": 1000000000000000019884624838656,
24
+ "model_specific_special_tokens": {
25
+ "audio_token": "<|audio|>",
26
+ "boa_token": "<|audio>",
27
+ "boi_token": "<|image>",
28
+ "eoa_token": "<audio|>",
29
+ "eoc_token": "<channel|>",
30
+ "eoi_token": "<image|>",
31
+ "eot_token": "<turn|>",
32
+ "escape_token": "<|\"|>",
33
+ "etc_token": "<tool_call|>",
34
+ "etd_token": "<tool|>",
35
+ "etr_token": "<tool_response|>",
36
+ "image_token": "<|image|>",
37
+ "soc_token": "<|channel>",
38
+ "sot_token": "<|turn>",
39
+ "stc_token": "<|tool_call>",
40
+ "std_token": "<|tool>",
41
+ "str_token": "<|tool_response>",
42
+ "think_token": "<|think|>"
43
+ },
44
+ "pad_token": "<pad>",
45
+ "padding_side": "right",
46
+ "processor_class": "Gemma4UnifiedProcessor",
47
+ "soc_token": "<|channel>",
48
+ "sot_token": "<|turn>",
49
+ "split_special_tokens": false,
50
+ "stc_token": "<|tool_call>",
51
+ "std_token": "<|tool>",
52
+ "str_token": "<|tool_response>",
53
+ "think_token": "<|think|>",
54
+ "tokenizer_class": "GemmaTokenizer",
55
+ "unk_token": "<unk>"
56
+ }
g1_cleanup_code/1000M/train_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 0.7863013698630137,
3
+ "total_flos": 5300110976090112.0,
4
+ "train_loss": 0.05387794186535783,
5
+ "train_runtime": 14746.3499,
6
+ "train_samples_per_second": 9.108,
7
+ "train_steps_per_second": 0.759
8
+ }
g1_cleanup_code/1000M/trainer_log.jsonl ADDED
@@ -0,0 +1,103 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"current_steps": 10180, "total_steps": 11193, "loss": 0.5852091312408447, "lr": 1.999957274187931e-05, "epoch": 0.7151387425360028, "percentage": 90.95, "elapsed_time": "0:01:01", "remaining_time": "0:00:06"}
2
+ {"current_steps": 10190, "total_steps": 11193, "loss": 0.624833059310913, "lr": 1.9991978062682906e-05, "epoch": 0.7158412363891816, "percentage": 91.04, "elapsed_time": "0:03:25", "remaining_time": "0:00:20"}
3
+ {"current_steps": 10200, "total_steps": 11193, "loss": 0.5865033626556396, "lr": 1.997489709987378e-05, "epoch": 0.7165437302423604, "percentage": 91.13, "elapsed_time": "0:05:49", "remaining_time": "0:00:34"}
4
+ {"current_steps": 10210, "total_steps": 11193, "loss": 0.6209458351135254, "lr": 1.9948346151502135e-05, "epoch": 0.7172462240955392, "percentage": 91.22, "elapsed_time": "0:08:14", "remaining_time": "0:00:47"}
5
+ {"current_steps": 10220, "total_steps": 11193, "loss": 0.5970417022705078, "lr": 1.991235055154325e-05, "epoch": 0.717948717948718, "percentage": 91.31, "elapsed_time": "0:10:38", "remaining_time": "0:01:00"}
6
+ {"current_steps": 10230, "total_steps": 11193, "loss": 0.6147752285003663, "lr": 1.9866944645724693e-05, "epoch": 0.7186512118018967, "percentage": 91.4, "elapsed_time": "0:13:02", "remaining_time": "0:01:13"}
7
+ {"current_steps": 10240, "total_steps": 11193, "loss": 0.6067847728729248, "lr": 1.9812171758754842e-05, "epoch": 0.7193537056550755, "percentage": 91.49, "elapsed_time": "0:15:26", "remaining_time": "0:01:26"}
8
+ {"current_steps": 10250, "total_steps": 11193, "loss": 0.6248685359954834, "lr": 1.9748084152983987e-05, "epoch": 0.7200561995082543, "percentage": 91.58, "elapsed_time": "0:17:52", "remaining_time": "0:01:38"}
9
+ {"current_steps": 10260, "total_steps": 11193, "loss": 0.6045065879821777, "lr": 1.967474297853744e-05, "epoch": 0.7207586933614331, "percentage": 91.66, "elapsed_time": "0:20:16", "remaining_time": "0:01:50"}
10
+ {"current_steps": 10270, "total_steps": 11193, "loss": 0.6001498222351074, "lr": 1.9592218214968256e-05, "epoch": 0.7214611872146118, "percentage": 91.75, "elapsed_time": "0:22:41", "remaining_time": "0:02:02"}
11
+ {"current_steps": 10280, "total_steps": 11193, "loss": 0.6187323570251465, "lr": 1.9500588604485213e-05, "epoch": 0.7221636810677906, "percentage": 91.84, "elapsed_time": "0:25:05", "remaining_time": "0:02:13"}
12
+ {"current_steps": 10290, "total_steps": 11193, "loss": 0.626680326461792, "lr": 1.939994157681979e-05, "epoch": 0.7228661749209695, "percentage": 91.93, "elapsed_time": "0:27:29", "remaining_time": "0:02:24"}
13
+ {"current_steps": 10300, "total_steps": 11193, "loss": 0.6163479804992675, "lr": 1.9290373165803803e-05, "epoch": 0.7235686687741483, "percentage": 92.02, "elapsed_time": "0:29:54", "remaining_time": "0:02:35"}
14
+ {"current_steps": 10310, "total_steps": 11193, "loss": 0.6144249916076661, "lr": 1.9171987917737328e-05, "epoch": 0.724271162627327, "percentage": 92.11, "elapsed_time": "0:32:18", "remaining_time": "0:02:46"}
15
+ {"current_steps": 10320, "total_steps": 11193, "loss": 0.6066095352172851, "lr": 1.9044898791634322e-05, "epoch": 0.7249736564805058, "percentage": 92.2, "elapsed_time": "0:34:43", "remaining_time": "0:02:56"}
16
+ {"current_steps": 10330, "total_steps": 11193, "loss": 0.6249143123626709, "lr": 1.8909227051441126e-05, "epoch": 0.7256761503336846, "percentage": 92.29, "elapsed_time": "0:37:07", "remaining_time": "0:03:06"}
17
+ {"current_steps": 10340, "total_steps": 11193, "loss": 0.6459973335266114, "lr": 1.8765102150330692e-05, "epoch": 0.7263786441868634, "percentage": 92.38, "elapsed_time": "0:39:32", "remaining_time": "0:03:15"}
18
+ {"current_steps": 10350, "total_steps": 11193, "loss": 0.6111915588378907, "lr": 1.861266160718297e-05, "epoch": 0.7270811380400422, "percentage": 92.47, "elapsed_time": "0:41:56", "remaining_time": "0:03:24"}
19
+ {"current_steps": 10360, "total_steps": 11193, "loss": 0.6064771175384521, "lr": 1.845205087536923e-05, "epoch": 0.7277836318932209, "percentage": 92.56, "elapsed_time": "0:44:20", "remaining_time": "0:03:33"}
20
+ {"current_steps": 10370, "total_steps": 11193, "loss": 0.6087264060974121, "lr": 1.8283423203965633e-05, "epoch": 0.7284861257463997, "percentage": 92.65, "elapsed_time": "0:46:44", "remaining_time": "0:03:42"}
21
+ {"current_steps": 10380, "total_steps": 11193, "loss": 0.598025131225586, "lr": 1.810693949152835e-05, "epoch": 0.7291886195995785, "percentage": 92.74, "elapsed_time": "0:49:09", "remaining_time": "0:03:50"}
22
+ {"current_steps": 10390, "total_steps": 11193, "loss": 0.5803035736083985, "lr": 1.7922768132569902e-05, "epoch": 0.7298911134527573, "percentage": 92.83, "elapsed_time": "0:51:34", "remaining_time": "0:03:59"}
23
+ {"current_steps": 10400, "total_steps": 11193, "loss": 0.6091340541839599, "lr": 1.773108485688302e-05, "epoch": 0.730593607305936, "percentage": 92.92, "elapsed_time": "0:53:58", "remaining_time": "0:04:06"}
24
+ {"current_steps": 10410, "total_steps": 11193, "loss": 0.6046764373779296, "lr": 1.7532072561865537e-05, "epoch": 0.7312961011591148, "percentage": 93.0, "elapsed_time": "0:56:23", "remaining_time": "0:04:14"}
25
+ {"current_steps": 10420, "total_steps": 11193, "loss": 0.60126953125, "lr": 1.732592113800616e-05, "epoch": 0.7319985950122937, "percentage": 93.09, "elapsed_time": "0:58:47", "remaining_time": "0:04:21"}
26
+ {"current_steps": 10430, "total_steps": 11193, "loss": 0.6024490356445312, "lr": 1.7112827287697668e-05, "epoch": 0.7327010888654725, "percentage": 93.18, "elapsed_time": "1:01:12", "remaining_time": "0:04:28"}
27
+ {"current_steps": 10440, "total_steps": 11193, "loss": 0.6132241249084472, "lr": 1.6892994337550452e-05, "epoch": 0.7334035827186512, "percentage": 93.27, "elapsed_time": "1:03:36", "remaining_time": "0:04:35"}
28
+ {"current_steps": 10450, "total_steps": 11193, "loss": 0.6058990478515625, "lr": 1.6666632044385487e-05, "epoch": 0.73410607657183, "percentage": 93.36, "elapsed_time": "1:06:01", "remaining_time": "0:04:41"}
29
+ {"current_steps": 10460, "total_steps": 11193, "loss": 0.6276742458343506, "lr": 1.6433956395091757e-05, "epoch": 0.7348085704250088, "percentage": 93.45, "elapsed_time": "1:08:26", "remaining_time": "0:04:47"}
30
+ {"current_steps": 10470, "total_steps": 11193, "loss": 0.6159543037414551, "lr": 1.6195189400539238e-05, "epoch": 0.7355110642781876, "percentage": 93.54, "elapsed_time": "1:10:50", "remaining_time": "0:04:53"}
31
+ {"current_steps": 10480, "total_steps": 11193, "loss": 0.5714602470397949, "lr": 1.595055888374398e-05, "epoch": 0.7362135581313664, "percentage": 93.63, "elapsed_time": "1:13:15", "remaining_time": "0:04:59"}
32
+ {"current_steps": 10490, "total_steps": 11193, "loss": 0.6032049179077148, "lr": 1.5700298262487434e-05, "epoch": 0.7369160519845451, "percentage": 93.72, "elapsed_time": "1:15:39", "remaining_time": "0:05:04"}
33
+ {"current_steps": 10500, "total_steps": 11193, "loss": 0.5805444717407227, "lr": 1.5444646326597467e-05, "epoch": 0.7376185458377239, "percentage": 93.81, "elapsed_time": "1:18:03", "remaining_time": "0:05:09"}
34
+ {"current_steps": 10510, "total_steps": 11193, "loss": 0.5913052558898926, "lr": 1.5183847010103586e-05, "epoch": 0.7383210396909027, "percentage": 93.9, "elapsed_time": "1:20:28", "remaining_time": "0:05:13"}
35
+ {"current_steps": 10520, "total_steps": 11193, "loss": 0.5918679714202881, "lr": 1.4918149158483686e-05, "epoch": 0.7390235335440815, "percentage": 93.99, "elapsed_time": "1:22:52", "remaining_time": "0:05:18"}
36
+ {"current_steps": 10530, "total_steps": 11193, "loss": 0.6032553672790527, "lr": 1.464780629122452e-05, "epoch": 0.7397260273972602, "percentage": 94.08, "elapsed_time": "1:25:18", "remaining_time": "0:05:22"}
37
+ {"current_steps": 10540, "total_steps": 11193, "loss": 0.5595372676849365, "lr": 1.4373076359922374e-05, "epoch": 0.740428521250439, "percentage": 94.17, "elapsed_time": "1:27:43", "remaining_time": "0:05:26"}
38
+ {"current_steps": 10550, "total_steps": 11193, "loss": 0.6196616649627685, "lr": 1.4094221502154774e-05, "epoch": 0.7411310151036179, "percentage": 94.26, "elapsed_time": "1:30:07", "remaining_time": "0:05:29"}
39
+ {"current_steps": 10560, "total_steps": 11193, "loss": 0.6003097057342529, "lr": 1.3811507791358113e-05, "epoch": 0.7418335089567967, "percentage": 94.34, "elapsed_time": "1:32:31", "remaining_time": "0:05:32"}
40
+ {"current_steps": 10570, "total_steps": 11193, "loss": 0.6162596702575683, "lr": 1.3525204982949753e-05, "epoch": 0.7425360028099754, "percentage": 94.43, "elapsed_time": "1:34:56", "remaining_time": "0:05:35"}
41
+ {"current_steps": 10580, "total_steps": 11193, "loss": 0.6030051708221436, "lr": 1.3235586256936996e-05, "epoch": 0.7432384966631542, "percentage": 94.52, "elapsed_time": "1:37:20", "remaining_time": "0:05:38"}
42
+ {"current_steps": 10590, "total_steps": 11193, "loss": 0.619550085067749, "lr": 1.294292795725836e-05, "epoch": 0.743940990516333, "percentage": 94.61, "elapsed_time": "1:39:44", "remaining_time": "0:05:40"}
43
+ {"current_steps": 10600, "total_steps": 11193, "loss": 0.591694974899292, "lr": 1.2647509328105978e-05, "epoch": 0.7446434843695118, "percentage": 94.7, "elapsed_time": "1:42:10", "remaining_time": "0:05:42"}
44
+ {"current_steps": 10610, "total_steps": 11193, "loss": 0.5779125213623046, "lr": 1.2349612247480686e-05, "epoch": 0.7453459782226906, "percentage": 94.79, "elapsed_time": "1:44:34", "remaining_time": "0:05:44"}
45
+ {"current_steps": 10620, "total_steps": 11193, "loss": 0.6079377174377442, "lr": 1.2049520958234e-05, "epoch": 0.7460484720758693, "percentage": 94.88, "elapsed_time": "1:46:58", "remaining_time": "0:05:46"}
46
+ {"current_steps": 10630, "total_steps": 11193, "loss": 0.5842973232269287, "lr": 1.1747521796853638e-05, "epoch": 0.7467509659290481, "percentage": 94.97, "elapsed_time": "1:49:22", "remaining_time": "0:05:47"}
47
+ {"current_steps": 10640, "total_steps": 11193, "loss": 0.611078405380249, "lr": 1.1443902920251395e-05, "epoch": 0.7474534597822269, "percentage": 95.06, "elapsed_time": "1:51:46", "remaining_time": "0:05:48"}
48
+ {"current_steps": 10650, "total_steps": 11193, "loss": 0.5831323146820069, "lr": 1.1138954030814009e-05, "epoch": 0.7481559536354057, "percentage": 95.15, "elapsed_time": "1:54:11", "remaining_time": "0:05:49"}
49
+ {"current_steps": 10660, "total_steps": 11193, "loss": 0.5817962646484375, "lr": 1.0832966099979388e-05, "epoch": 0.7488584474885844, "percentage": 95.24, "elapsed_time": "1:56:47", "remaining_time": "0:05:50"}
50
+ {"current_steps": 10670, "total_steps": 11193, "loss": 0.587282133102417, "lr": 1.0526231090601969e-05, "epoch": 0.7495609413417632, "percentage": 95.33, "elapsed_time": "1:59:11", "remaining_time": "0:05:50"}
51
+ {"current_steps": 10680, "total_steps": 11193, "loss": 0.5874773979187011, "lr": 1.0219041678372106e-05, "epoch": 0.7502634351949421, "percentage": 95.42, "elapsed_time": "2:01:36", "remaining_time": "0:05:50"}
52
+ {"current_steps": 10690, "total_steps": 11193, "loss": 0.5797173976898193, "lr": 9.911690972555268e-06, "epoch": 0.7509659290481209, "percentage": 95.51, "elapsed_time": "2:04:00", "remaining_time": "0:05:50"}
53
+ {"current_steps": 10700, "total_steps": 11193, "loss": 0.6123420238494873, "lr": 9.604472236317541e-06, "epoch": 0.7516684229012996, "percentage": 95.6, "elapsed_time": "2:06:24", "remaining_time": "0:05:49"}
54
+ {"current_steps": 10710, "total_steps": 11193, "loss": 0.5702229976654053, "lr": 9.297678606904335e-06, "epoch": 0.7523709167544784, "percentage": 95.68, "elapsed_time": "2:08:49", "remaining_time": "0:05:48"}
55
+ {"current_steps": 10720, "total_steps": 11193, "loss": 0.5769252777099609, "lr": 8.991602815939155e-06, "epoch": 0.7530734106076572, "percentage": 95.77, "elapsed_time": "2:11:13", "remaining_time": "0:05:47"}
56
+ {"current_steps": 10730, "total_steps": 11193, "loss": 0.5780287742614746, "lr": 8.686536910109464e-06, "epoch": 0.753775904460836, "percentage": 95.86, "elapsed_time": "2:13:37", "remaining_time": "0:05:45"}
57
+ {"current_steps": 10740, "total_steps": 11193, "loss": 0.6100452423095704, "lr": 8.382771972506084e-06, "epoch": 0.7544783983140148, "percentage": 95.95, "elapsed_time": "2:16:02", "remaining_time": "0:05:44"}
58
+ {"current_steps": 10750, "total_steps": 11193, "loss": 0.5884391784667968, "lr": 8.08059784488198e-06, "epoch": 0.7551808921671935, "percentage": 96.04, "elapsed_time": "2:18:27", "remaining_time": "0:05:42"}
59
+ {"current_steps": 10760, "total_steps": 11193, "loss": 0.5781902313232422, "lr": 7.780302851095544e-06, "epoch": 0.7558833860203723, "percentage": 96.13, "elapsed_time": "2:20:52", "remaining_time": "0:05:40"}
60
+ {"current_steps": 10770, "total_steps": 11193, "loss": 0.5921500205993653, "lr": 7.482173522002166e-06, "epoch": 0.7565858798735511, "percentage": 96.22, "elapsed_time": "2:23:16", "remaining_time": "0:05:37"}
61
+ {"current_steps": 10780, "total_steps": 11193, "loss": 0.6058819770812989, "lr": 7.186494322056624e-06, "epoch": 0.7572883737267299, "percentage": 96.31, "elapsed_time": "2:25:40", "remaining_time": "0:05:34"}
62
+ {"current_steps": 10790, "total_steps": 11193, "loss": 0.6233699798583985, "lr": 6.893547377887183e-06, "epoch": 0.7579908675799086, "percentage": 96.4, "elapsed_time": "2:28:04", "remaining_time": "0:05:31"}
63
+ {"current_steps": 10800, "total_steps": 11193, "loss": 0.5790233612060547, "lr": 6.6036122091003365e-06, "epoch": 0.7586933614330874, "percentage": 96.49, "elapsed_time": "2:30:29", "remaining_time": "0:05:28"}
64
+ {"current_steps": 10810, "total_steps": 11193, "loss": 0.5863173484802247, "lr": 6.316965461573097e-06, "epoch": 0.7593958552862663, "percentage": 96.58, "elapsed_time": "2:32:54", "remaining_time": "0:05:25"}
65
+ {"current_steps": 10820, "total_steps": 11193, "loss": 0.592318344116211, "lr": 6.0338806434872965e-06, "epoch": 0.7600983491394451, "percentage": 96.67, "elapsed_time": "2:35:39", "remaining_time": "0:05:21"}
66
+ {"current_steps": 10830, "total_steps": 11193, "loss": 0.5882054805755615, "lr": 5.754627864357749e-06, "epoch": 0.7608008429926238, "percentage": 96.76, "elapsed_time": "2:38:03", "remaining_time": "0:05:17"}
67
+ {"current_steps": 10840, "total_steps": 11193, "loss": 0.5774524688720704, "lr": 5.479473577303319e-06, "epoch": 0.7615033368458026, "percentage": 96.85, "elapsed_time": "2:40:28", "remaining_time": "0:05:13"}
68
+ {"current_steps": 10850, "total_steps": 11193, "loss": 0.5922804832458496, "lr": 5.208680324806778e-06, "epoch": 0.7622058306989814, "percentage": 96.94, "elapsed_time": "2:42:52", "remaining_time": "0:05:08"}
69
+ {"current_steps": 10860, "total_steps": 11193, "loss": 0.570035171508789, "lr": 4.94250648820606e-06, "epoch": 0.7629083245521602, "percentage": 97.02, "elapsed_time": "2:45:16", "remaining_time": "0:05:04"}
70
+ {"current_steps": 10870, "total_steps": 11193, "loss": 0.5976253509521484, "lr": 4.681206041155951e-06, "epoch": 0.763610818405339, "percentage": 97.11, "elapsed_time": "2:47:40", "remaining_time": "0:04:58"}
71
+ {"current_steps": 10880, "total_steps": 11193, "loss": 0.5517823219299316, "lr": 4.42502830729538e-06, "epoch": 0.7643133122585177, "percentage": 97.2, "elapsed_time": "2:50:05", "remaining_time": "0:04:53"}
72
+ {"current_steps": 10890, "total_steps": 11193, "loss": 0.6046184539794922, "lr": 4.174217722351669e-06, "epoch": 0.7650158061116965, "percentage": 97.29, "elapsed_time": "2:52:30", "remaining_time": "0:04:47"}
73
+ {"current_steps": 10900, "total_steps": 11193, "loss": 0.5755278587341308, "lr": 3.929013600908575e-06, "epoch": 0.7657182999648753, "percentage": 97.38, "elapsed_time": "2:54:54", "remaining_time": "0:04:42"}
74
+ {"current_steps": 10910, "total_steps": 11193, "loss": 0.5779466152191162, "lr": 3.689649908060807e-06, "epoch": 0.7664207938180541, "percentage": 97.47, "elapsed_time": "2:57:18", "remaining_time": "0:04:35"}
75
+ {"current_steps": 10920, "total_steps": 11193, "loss": 0.5788602352142334, "lr": 3.4563550361728028e-06, "epoch": 0.7671232876712328, "percentage": 97.56, "elapsed_time": "2:59:43", "remaining_time": "0:04:29"}
76
+ {"current_steps": 10930, "total_steps": 11193, "loss": 0.5751082420349121, "lr": 3.2293515869548216e-06, "epoch": 0.7678257815244116, "percentage": 97.65, "elapsed_time": "3:02:07", "remaining_time": "0:04:22"}
77
+ {"current_steps": 10940, "total_steps": 11193, "loss": 0.5937278270721436, "lr": 3.0088561590642746e-06, "epoch": 0.7685282753775905, "percentage": 97.74, "elapsed_time": "3:04:31", "remaining_time": "0:04:16"}
78
+ {"current_steps": 10950, "total_steps": 11193, "loss": 0.5773450374603272, "lr": 2.795079141434941e-06, "epoch": 0.7692307692307693, "percentage": 97.83, "elapsed_time": "3:06:56", "remaining_time": "0:04:08"}
79
+ {"current_steps": 10960, "total_steps": 11193, "loss": 0.5806230545043946, "lr": 2.588224512531299e-06, "epoch": 0.769933263083948, "percentage": 97.92, "elapsed_time": "3:09:21", "remaining_time": "0:04:01"}
80
+ {"current_steps": 10970, "total_steps": 11193, "loss": 0.5859660625457763, "lr": 2.388489645719495e-06, "epoch": 0.7706357569371268, "percentage": 98.01, "elapsed_time": "3:11:45", "remaining_time": "0:03:53"}
81
+ {"current_steps": 10980, "total_steps": 11193, "loss": 0.5792638301849365, "lr": 2.196065120940663e-06, "epoch": 0.7713382507903056, "percentage": 98.1, "elapsed_time": "3:14:09", "remaining_time": "0:03:45"}
82
+ {"current_steps": 10990, "total_steps": 11193, "loss": 0.5705962181091309, "lr": 2.0111345428662808e-06, "epoch": 0.7720407446434844, "percentage": 98.19, "elapsed_time": "3:16:33", "remaining_time": "0:03:37"}
83
+ {"current_steps": 11000, "total_steps": 11193, "loss": 0.5686861991882324, "lr": 1.8338743657091102e-06, "epoch": 0.7727432384966632, "percentage": 98.28, "elapsed_time": "3:18:58", "remaining_time": "0:03:29"}
84
+ {"current_steps": 11010, "total_steps": 11193, "loss": 0.5631584167480469, "lr": 1.6644537248568113e-06, "epoch": 0.7734457323498419, "percentage": 98.37, "elapsed_time": "3:21:22", "remaining_time": "0:03:20"}
85
+ {"current_steps": 11020, "total_steps": 11193, "loss": 0.5635076045989991, "lr": 1.503034275488951e-06, "epoch": 0.7741482262030207, "percentage": 98.45, "elapsed_time": "3:23:46", "remaining_time": "0:03:11"}
86
+ {"current_steps": 11030, "total_steps": 11193, "loss": 0.5801867961883544, "lr": 1.3497700383313495e-06, "epoch": 0.7748507200561995, "percentage": 98.54, "elapsed_time": "3:26:11", "remaining_time": "0:03:02"}
87
+ {"current_steps": 11040, "total_steps": 11193, "loss": 0.5985492706298828, "lr": 1.2048072526949562e-06, "epoch": 0.7755532139093783, "percentage": 98.63, "elapsed_time": "3:28:35", "remaining_time": "0:02:53"}
88
+ {"current_steps": 11050, "total_steps": 11193, "loss": 0.5673690795898437, "lr": 1.0682842369394756e-06, "epoch": 0.776255707762557, "percentage": 98.72, "elapsed_time": "3:31:00", "remaining_time": "0:02:43"}
89
+ {"current_steps": 11060, "total_steps": 11193, "loss": 0.6037036895751953, "lr": 9.403312564948792e-07, "epoch": 0.7769582016157358, "percentage": 98.81, "elapsed_time": "3:33:24", "remaining_time": "0:02:33"}
90
+ {"current_steps": 11070, "total_steps": 11193, "loss": 0.5852627754211426, "lr": 8.210703995667471e-07, "epoch": 0.7776606954689147, "percentage": 98.9, "elapsed_time": "3:35:48", "remaining_time": "0:02:23"}
91
+ {"current_steps": 11080, "total_steps": 11193, "loss": 0.5962277412414551, "lr": 7.106154606440245e-07, "epoch": 0.7783631893220935, "percentage": 98.99, "elapsed_time": "3:38:12", "remaining_time": "0:02:13"}
92
+ {"current_steps": 11090, "total_steps": 11193, "loss": 0.5665205955505371, "lr": 6.090718319203393e-07, "epoch": 0.7790656831752722, "percentage": 99.08, "elapsed_time": "3:40:37", "remaining_time": "0:02:02"}
93
+ {"current_steps": 11100, "total_steps": 11193, "loss": 0.5832221984863282, "lr": 5.165364027325089e-07, "epoch": 0.779768177028451, "percentage": 99.17, "elapsed_time": "3:43:02", "remaining_time": "0:01:52"}
94
+ {"current_steps": 11110, "total_steps": 11193, "loss": 0.5794486999511719, "lr": 4.330974671121585e-07, "epoch": 0.7804706708816298, "percentage": 99.26, "elapsed_time": "3:45:26", "remaining_time": "0:01:41"}
95
+ {"current_steps": 11120, "total_steps": 11193, "loss": 0.5899890422821045, "lr": 3.588346395386846e-07, "epoch": 0.7811731647348086, "percentage": 99.35, "elapsed_time": "3:47:50", "remaining_time": "0:01:29"}
96
+ {"current_steps": 11130, "total_steps": 11193, "loss": 0.5896859645843506, "lr": 2.938187789739395e-07, "epoch": 0.7818756585879874, "percentage": 99.44, "elapsed_time": "3:50:15", "remaining_time": "0:01:18"}
97
+ {"current_steps": 11140, "total_steps": 11193, "loss": 0.5642678737640381, "lr": 2.3811192125110724e-07, "epoch": 0.7825781524411661, "percentage": 99.53, "elapsed_time": "3:52:39", "remaining_time": "0:01:06"}
98
+ {"current_steps": 11150, "total_steps": 11193, "loss": 0.5775588512420654, "lr": 1.917672198823177e-07, "epoch": 0.7832806462943449, "percentage": 99.62, "elapsed_time": "3:55:03", "remaining_time": "0:00:54"}
99
+ {"current_steps": 11160, "total_steps": 11193, "loss": 0.5753410339355469, "lr": 1.5482889534143795e-07, "epoch": 0.7839831401475237, "percentage": 99.71, "elapsed_time": "3:57:28", "remaining_time": "0:00:42"}
100
+ {"current_steps": 11170, "total_steps": 11193, "loss": 0.5725598335266113, "lr": 1.273321928704694e-07, "epoch": 0.7846856340007025, "percentage": 99.79, "elapsed_time": "3:59:53", "remaining_time": "0:00:29"}
101
+ {"current_steps": 11180, "total_steps": 11193, "loss": 0.5798064231872558, "lr": 1.0930334884977866e-07, "epoch": 0.7853881278538812, "percentage": 99.88, "elapsed_time": "4:02:17", "remaining_time": "0:00:16"}
102
+ {"current_steps": 11190, "total_steps": 11193, "loss": 0.5891654014587402, "lr": 1.0075956576427487e-07, "epoch": 0.78609062170706, "percentage": 99.97, "elapsed_time": "4:04:42", "remaining_time": "0:00:03"}
103
+ {"current_steps": 11193, "total_steps": 11193, "epoch": 0.7863013698630137, "percentage": 100.0, "elapsed_time": "4:05:46", "remaining_time": "0:00:00"}
g1_cleanup_code/1000M/trainer_state.json ADDED
The diff for this file is too large to render. See raw diff
 
g1_cleanup_code/1000M/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:24e8dd0aea22bac7539678a831f485e6dd20b1816e427788691df9d9de1c0ad9
3
+ size 8209
g1_cleanup_code/1000M/training_loss.png ADDED