jadhavmanasi70 commited on
Commit
7df51ac
·
verified ·
1 Parent(s): bfbaf7a

Add 11 files

Browse files
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: meta-llama/Llama-4-Scout-17B-16E-Instruct
3
+ library_name: peft
4
+ license: other
5
+ tags:
6
+ - lora
7
+ - peft
8
+ - adapter
9
+ - adaption
10
+ ---
11
+
12
+ # adaption_chart_misleading_patterns
13
+
14
+ ## Model Training
15
+
16
+ A LORA adapter for `meta-llama/Llama-4-Scout-17B-16E-Instruct`. This model was trained with SFT using [Adaption](https://adaptionlabs.ai)'s AutoScientist on the chart_misleading_patterns dataset.
17
+
18
+
19
+ ![Training metrics](training-metrics.png)
20
+
21
+ ### AutoScientist Config
22
+
23
+ ```json
24
+ {
25
+ "job_id": "16f4d621-cb1b-4a32-9d3b-70a2a15811ac",
26
+ "training_experiment_id": "bcbc06cf-a5c1-4ed1-a70a-75523da76408",
27
+ "original_model_name": "meta-llama/Llama-4-Scout-17B-16E-Instruct",
28
+ "trained_model_name": "adaption_chart_misleading_patterns",
29
+ "training_method": "sft",
30
+ "training_type": "lora",
31
+ "data_format": "chat",
32
+ "hyperparams": {
33
+ "lora": "true",
34
+ "lora_r": 64,
35
+ "n_evals": 5,
36
+ "n_epochs": 4,
37
+ "batch_size": "max",
38
+ "lora_alpha": 128,
39
+ "lora_dropout": 0,
40
+ "min_lr_ratio": 0.1,
41
+ "warmup_ratio": 0.03,
42
+ "weight_decay": 0.02,
43
+ "learning_rate": 0.0001,
44
+ "max_grad_norm": 1,
45
+ "base_model_size": "109B",
46
+ "train_on_inputs": "false",
47
+ "training_method": "sft",
48
+ "lr_scheduler_type": "cosine",
49
+ "scheduler_num_cycles": 0.5,
50
+ "lora_trainable_modules": "k_proj,o_proj,q_proj,v_proj,shared_expert.gate_proj,shared_expert.up_proj,shared_expert.down_proj,feed_forward.gate_proj,feed_forward.up_proj,feed_forward.down_proj"
51
+ }
52
+ }
53
+ ```
54
+
55
+ ## Training Data
56
+
57
+ The model was trained on 27,585 rows of adapted data with the following domain distribution: data-analysis-visualization (100%), market-analysis (0%).
58
+
59
+ ## Model Evaluation
60
+
61
+ The model was evaluated on an in-distribution held-out test set as well as a broader domain-specific test set to measure generalization.
62
+
63
+
64
+ ![Win rates](win-rates.png)
65
+
66
+ | Domain | Win rate vs. base model |
67
+ | --- | --- |
68
+ | data-analysis-visualization | 82% |
69
+
70
+ ## How to use
71
+
72
+ ```bash
73
+ pip install torch transformers peft
74
+ ```
75
+
76
+ ```python
77
+ import torch
78
+ from transformers import AutoModelForCausalLM, AutoTokenizer
79
+ from peft import PeftModel
80
+
81
+ BASE = "meta-llama/Llama-4-Scout-17B-16E-Instruct"
82
+ ADAPTER = "<this-repo-id>"
83
+
84
+ device = "cuda" if torch.cuda.is_available() else "cpu"
85
+ dtype = torch.float32 if device == "cpu" else torch.bfloat16
86
+
87
+ base = AutoModelForCausalLM.from_pretrained(BASE, dtype=dtype).to(device)
88
+ model = PeftModel.from_pretrained(base, ADAPTER)
89
+ # Optional: merge the LoRA weights into the base for faster inference
90
+ model = model.merge_and_unload()
91
+ model.eval()
92
+
93
+ tokenizer = AutoTokenizer.from_pretrained(BASE)
94
+ messages = [{"role": "user", "content": "Hello!"}]
95
+ text = tokenizer.apply_chat_template(
96
+ messages, tokenize=False, add_generation_prompt=True)
97
+ inputs = tokenizer(text, return_tensors="pt").to(device)
98
+
99
+ with torch.inference_mode():
100
+ out = model.generate(**inputs, max_new_tokens=512)
101
+ print(tokenizer.decode(out[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True))
102
+ ```
adapter_config.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "togethercomputer/Llama-4-Scout-17B-16E-Instruct_bnb_4bit",
5
+ "bias": "none",
6
+ "corda_config": null,
7
+ "eva_config": null,
8
+ "exclude_modules": [],
9
+ "fan_in_fan_out": false,
10
+ "inference_mode": true,
11
+ "init_lora_weights": true,
12
+ "layer_replication": null,
13
+ "layers_pattern": null,
14
+ "layers_to_transform": null,
15
+ "loftq_config": {},
16
+ "lora_alpha": 128,
17
+ "lora_bias": false,
18
+ "lora_dropout": 0.0,
19
+ "megatron_config": null,
20
+ "megatron_core": "megatron.core",
21
+ "modules_to_save": null,
22
+ "peft_type": "LORA",
23
+ "r": 64,
24
+ "rank_pattern": {},
25
+ "revision": null,
26
+ "target_modules": [
27
+ "v_proj",
28
+ "feed_forward.gate_proj",
29
+ "k_proj",
30
+ "shared_expert.gate_proj",
31
+ "feed_forward.up_proj",
32
+ "q_proj",
33
+ "shared_expert.up_proj",
34
+ "shared_expert.down_proj",
35
+ "o_proj",
36
+ "feed_forward.down_proj"
37
+ ],
38
+ "task_type": "CAUSAL_LM",
39
+ "trainable_token_indices": null,
40
+ "use_dora": false,
41
+ "use_rslora": false
42
+ }
adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:99108b26e56aec520b08c7ca79b9034c2d4a189ae37ba5f2202d472ded2fa400
3
+ size 893484120
chat_template.jinja ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {{- bos_token }}
2
+ {%- if custom_tools is defined %}
3
+ {%- set tools = custom_tools %}
4
+ {%- endif %}
5
+ {%- if not tools_in_user_message is defined %}
6
+ {%- set tools_in_user_message = true %}
7
+ {%- endif %}
8
+ {%- if not date_string is defined %}
9
+ {%- if strftime_now is defined %}
10
+ {%- set date_string = strftime_now("%d %b %Y") %}
11
+ {%- else %}
12
+ {%- set date_string = "26 Jul 2024" %}
13
+ {%- endif %}
14
+ {%- endif %}
15
+ {%- if not tools is defined %}
16
+ {%- set tools = none %}
17
+ {%- endif %}
18
+
19
+ {#- This block extracts the system message, so we can slot it into the right place. #}
20
+ {%- if messages[0]['role'] == 'system' %}
21
+ {%- if messages[0]['content'] is string %}
22
+ {%- set system_message = messages[0]['content']|trim %}
23
+ {%- else %}
24
+ {#- FIXME: The processor requires an array, always. #}
25
+ {%- set system_message = messages[0]['content'][0]['text']|trim %}
26
+ {%- endif %}
27
+ {%- set messages = messages[1:] %}
28
+ {%- set user_supplied_system_message = true %}
29
+ {%- else %}
30
+ {%- set system_message = "" %}
31
+ {%- set user_supplied_system_message = false %}
32
+ {%- endif %}
33
+
34
+ {#- System message if the user supplied one #}
35
+ {%- if user_supplied_system_message %}
36
+ {{- "<|header_start|>system<|header_end|>\n\n" }}
37
+ {%- if tools is not none %}
38
+ {{- "Environment: ipython\n" }}
39
+ {%- endif %}
40
+ {%- if tools is not none and not tools_in_user_message %}
41
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
42
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
43
+ {{- "Do not use variables.\n\n" }}
44
+ {%- for t in tools %}
45
+ {{- t | tojson(indent=4) }}
46
+ {{- "\n\n" }}
47
+ {%- endfor %}
48
+ {%- endif %}
49
+ {{- system_message }}
50
+ {{- "<|eot|>" }}
51
+ {%- endif %}
52
+
53
+ {#- Custom tools are passed in a user message with some extra guidance #}
54
+ {%- if tools_in_user_message and not tools is none %}
55
+ {#- Extract the first user message so we can plug it in here #}
56
+ {%- if messages | length != 0 %}
57
+ {%- set first_user_message = messages[0]['content']|trim %}
58
+ {%- set messages = messages[1:] %}
59
+ {%- else %}
60
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
61
+ {%- endif %}
62
+ {{- '<|header_start|>user<|header_end|>\n\n' -}}
63
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
64
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
65
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
66
+ {{- "Do not use variables.\n\n" }}
67
+ {%- for t in tools %}
68
+ {{- t | tojson(indent=4) }}
69
+ {{- "\n\n" }}
70
+ {%- endfor %}
71
+ {{- first_user_message + "<|eot|>"}}
72
+ {%- endif %}
73
+
74
+ {%- for message in messages %}
75
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
76
+ {{- '<|header_start|>' + message['role'] + '<|header_end|>\n\n' }}
77
+ {%- if message['content'] is string %}
78
+ {{- message['content'] }}
79
+ {%- else %}
80
+ {%- for content in message['content'] %}
81
+ {%- if content['type'] == 'image' %}
82
+ {{- '<|image|>' }}
83
+ {%- elif content['type'] == 'text' %}
84
+ {{- content['text'] }}
85
+ {%- endif %}
86
+ {%- endfor %}
87
+ {%- endif %}
88
+ {{- "<|eot|>" }}
89
+ {%- elif 'tool_calls' in message and message.tool_calls|length > 0 %}
90
+ {{- '<|header_start|>assistant<|header_end|>\n\n' -}}
91
+ {{- '<|python_start|>' }}
92
+ {%- if message['content'] is string %}
93
+ {{- message['content'] }}
94
+ {%- else %}
95
+ {%- for content in message['content'] %}
96
+ {%- if content['type'] == 'image' %}
97
+ {{- '<|image|>' }}
98
+ {%- elif content['type'] == 'text' %}
99
+ {{- content['text'] }}
100
+ {%- endif %}
101
+ {%- endfor %}
102
+ {%- endif %}
103
+ {{- '<|python_end|>' }}
104
+ {%- for tool_call in message.tool_calls %}
105
+ {{- '{"name": "' + tool_call.function.name + '", ' }}
106
+ {{- '"parameters": ' }}
107
+ {{- tool_call.function.arguments | tojson }}
108
+ {{- "}" }}
109
+ {%- endfor %}
110
+ {{- "<|eot|>" }}
111
+ {%- elif message.role == "tool" or message.role == "ipython" %}
112
+ {{- "<|header_start|>ipython<|header_end|>\n\n" }}
113
+ {%- if message.content is mapping or message.content is iterable %}
114
+ {{- message.content | tojson }}
115
+ {%- else %}
116
+ {{- message.content }}
117
+ {%- endif %}
118
+ {{- "<|eot|>" }}
119
+ {%- endif %}
120
+ {%- endfor %}
121
+ {%- if add_generation_prompt %}
122
+ {{- '<|header_start|>assistant<|header_end|>\n\n' }}
123
+ {%- endif %}
config.json ADDED
@@ -0,0 +1,215 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Llama4ForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_chunk_size": 8192,
7
+ "attention_dropout": 0.0,
8
+ "attn_scale": 0.1,
9
+ "attn_temperature_tuning": true,
10
+ "bos_token_id": 200000,
11
+ "cache_implementation": "hybrid",
12
+ "dtype": "bfloat16",
13
+ "eos_token_id": 200008,
14
+ "floor_scale": 8192,
15
+ "for_llm_compressor": false,
16
+ "head_dim": 128,
17
+ "hidden_act": "silu",
18
+ "hidden_size": 5120,
19
+ "initializer_range": 0.02,
20
+ "interleave_moe_layer_step": 1,
21
+ "intermediate_size": 8192,
22
+ "intermediate_size_mlp": 16384,
23
+ "layer_types": [
24
+ "chunked_attention",
25
+ "chunked_attention",
26
+ "chunked_attention",
27
+ "full_attention",
28
+ "chunked_attention",
29
+ "chunked_attention",
30
+ "chunked_attention",
31
+ "full_attention",
32
+ "chunked_attention",
33
+ "chunked_attention",
34
+ "chunked_attention",
35
+ "full_attention",
36
+ "chunked_attention",
37
+ "chunked_attention",
38
+ "chunked_attention",
39
+ "full_attention",
40
+ "chunked_attention",
41
+ "chunked_attention",
42
+ "chunked_attention",
43
+ "full_attention",
44
+ "chunked_attention",
45
+ "chunked_attention",
46
+ "chunked_attention",
47
+ "full_attention",
48
+ "chunked_attention",
49
+ "chunked_attention",
50
+ "chunked_attention",
51
+ "full_attention",
52
+ "chunked_attention",
53
+ "chunked_attention",
54
+ "chunked_attention",
55
+ "full_attention",
56
+ "chunked_attention",
57
+ "chunked_attention",
58
+ "chunked_attention",
59
+ "full_attention",
60
+ "chunked_attention",
61
+ "chunked_attention",
62
+ "chunked_attention",
63
+ "full_attention",
64
+ "chunked_attention",
65
+ "chunked_attention",
66
+ "chunked_attention",
67
+ "full_attention",
68
+ "chunked_attention",
69
+ "chunked_attention",
70
+ "chunked_attention",
71
+ "full_attention"
72
+ ],
73
+ "max_position_embeddings": 10485760,
74
+ "model_type": "llama4_text",
75
+ "moe_layers": [
76
+ 0,
77
+ 1,
78
+ 2,
79
+ 3,
80
+ 4,
81
+ 5,
82
+ 6,
83
+ 7,
84
+ 8,
85
+ 9,
86
+ 10,
87
+ 11,
88
+ 12,
89
+ 13,
90
+ 14,
91
+ 15,
92
+ 16,
93
+ 17,
94
+ 18,
95
+ 19,
96
+ 20,
97
+ 21,
98
+ 22,
99
+ 23,
100
+ 24,
101
+ 25,
102
+ 26,
103
+ 27,
104
+ 28,
105
+ 29,
106
+ 30,
107
+ 31,
108
+ 32,
109
+ 33,
110
+ 34,
111
+ 35,
112
+ 36,
113
+ 37,
114
+ 38,
115
+ 39,
116
+ 40,
117
+ 41,
118
+ 42,
119
+ 43,
120
+ 44,
121
+ 45,
122
+ 46,
123
+ 47
124
+ ],
125
+ "no_rope_layer_interval": 4,
126
+ "no_rope_layers": [
127
+ 1,
128
+ 1,
129
+ 1,
130
+ 0,
131
+ 1,
132
+ 1,
133
+ 1,
134
+ 0,
135
+ 1,
136
+ 1,
137
+ 1,
138
+ 0,
139
+ 1,
140
+ 1,
141
+ 1,
142
+ 0,
143
+ 1,
144
+ 1,
145
+ 1,
146
+ 0,
147
+ 1,
148
+ 1,
149
+ 1,
150
+ 0,
151
+ 1,
152
+ 1,
153
+ 1,
154
+ 0,
155
+ 1,
156
+ 1,
157
+ 1,
158
+ 0,
159
+ 1,
160
+ 1,
161
+ 1,
162
+ 0,
163
+ 1,
164
+ 1,
165
+ 1,
166
+ 0,
167
+ 1,
168
+ 1,
169
+ 1,
170
+ 0,
171
+ 1,
172
+ 1,
173
+ 1,
174
+ 0
175
+ ],
176
+ "num_attention_heads": 40,
177
+ "num_experts_per_tok": 1,
178
+ "num_hidden_layers": 48,
179
+ "num_key_value_heads": 8,
180
+ "num_local_experts": 16,
181
+ "output_router_logits": false,
182
+ "pad_token_id": 200018,
183
+ "quantization_config": {
184
+ "_load_in_4bit": true,
185
+ "_load_in_8bit": false,
186
+ "bnb_4bit_compute_dtype": "bfloat16",
187
+ "bnb_4bit_quant_storage": "bfloat16",
188
+ "bnb_4bit_quant_type": "nf4",
189
+ "bnb_4bit_use_double_quant": false,
190
+ "llm_int8_enable_fp32_cpu_offload": false,
191
+ "llm_int8_has_fp16_weight": false,
192
+ "llm_int8_skip_modules": null,
193
+ "llm_int8_threshold": 6.0,
194
+ "load_in_4bit": true,
195
+ "load_in_8bit": false,
196
+ "quant_method": "bitsandbytes"
197
+ },
198
+ "rms_norm_eps": 1e-05,
199
+ "rope_parameters": {
200
+ "factor": 16.0,
201
+ "high_freq_factor": 1.0,
202
+ "low_freq_factor": 1.0,
203
+ "original_max_position_embeddings": 8192,
204
+ "rope_theta": 500000.0,
205
+ "rope_type": "llama3"
206
+ },
207
+ "router_aux_loss_coef": 0.001,
208
+ "router_jitter_noise": 0.0,
209
+ "tie_word_embeddings": false,
210
+ "transformers_version": "5.13.0",
211
+ "use_cache": false,
212
+ "use_qk_norm": true,
213
+ "vocab_size": 202048,
214
+ "torch_dtype": "bfloat16"
215
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<|begin_of_text|>",
3
+ "eos_token": "<|eot|>",
4
+ "pad_token": "<|finetune_right_pad|>"
5
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:172c9eb4beafc72601690da3ccfcede5c2e6806a8d5ec1fca33e22acea8023a4
3
+ size 27948578
tokenizer_config.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "bos_token": "<|begin_of_text|>",
4
+ "clean_up_tokenization_spaces": false,
5
+ "eos_token": "<|eot|>",
6
+ "is_local": false,
7
+ "local_files_only": true,
8
+ "model_input_names": [
9
+ "input_ids",
10
+ "attention_mask"
11
+ ],
12
+ "model_max_length": 10485760,
13
+ "pad_token": "<|finetune_right_pad|>",
14
+ "padding_side": "right",
15
+ "processor_class": "Llama4Processor",
16
+ "tokenizer_class": "TokenizersBackend"
17
+ }
trainer_state.json ADDED
@@ -0,0 +1,1054 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 4.0,
6
+ "eval_steps": 28,
7
+ "global_step": 140,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.02857142857142857,
14
+ "grad_norm": 0.9386152625083923,
15
+ "learning_rate": 0.0,
16
+ "loss": 1.744140625,
17
+ "step": 1
18
+ },
19
+ {
20
+ "epoch": 0.05714285714285714,
21
+ "grad_norm": 0.852827250957489,
22
+ "learning_rate": 2e-05,
23
+ "loss": 1.6787109375,
24
+ "step": 2
25
+ },
26
+ {
27
+ "epoch": 0.08571428571428572,
28
+ "grad_norm": 0.8461427092552185,
29
+ "learning_rate": 4e-05,
30
+ "loss": 1.671875,
31
+ "step": 3
32
+ },
33
+ {
34
+ "epoch": 0.11428571428571428,
35
+ "grad_norm": 0.6573083400726318,
36
+ "learning_rate": 6e-05,
37
+ "loss": 1.5703125,
38
+ "step": 4
39
+ },
40
+ {
41
+ "epoch": 0.14285714285714285,
42
+ "grad_norm": 0.5101009011268616,
43
+ "learning_rate": 8e-05,
44
+ "loss": 1.5732421875,
45
+ "step": 5
46
+ },
47
+ {
48
+ "epoch": 0.17142857142857143,
49
+ "grad_norm": 0.3327346444129944,
50
+ "learning_rate": 0.0001,
51
+ "loss": 1.5615234375,
52
+ "step": 6
53
+ },
54
+ {
55
+ "epoch": 0.2,
56
+ "grad_norm": 0.5101948380470276,
57
+ "learning_rate": 9.998781585307577e-05,
58
+ "loss": 1.41162109375,
59
+ "step": 7
60
+ },
61
+ {
62
+ "epoch": 0.22857142857142856,
63
+ "grad_norm": 0.6675843000411987,
64
+ "learning_rate": 9.99512700102336e-05,
65
+ "loss": 1.41015625,
66
+ "step": 8
67
+ },
68
+ {
69
+ "epoch": 0.2571428571428571,
70
+ "grad_norm": 0.6396194100379944,
71
+ "learning_rate": 9.989038226169209e-05,
72
+ "loss": 1.474609375,
73
+ "step": 9
74
+ },
75
+ {
76
+ "epoch": 0.2857142857142857,
77
+ "grad_norm": 0.5296529531478882,
78
+ "learning_rate": 9.98051855792412e-05,
79
+ "loss": 1.4306640625,
80
+ "step": 10
81
+ },
82
+ {
83
+ "epoch": 0.3142857142857143,
84
+ "grad_norm": 0.2675076723098755,
85
+ "learning_rate": 9.969572609838744e-05,
86
+ "loss": 1.2821044921875,
87
+ "step": 11
88
+ },
89
+ {
90
+ "epoch": 0.34285714285714286,
91
+ "grad_norm": 0.2704640030860901,
92
+ "learning_rate": 9.956206309337068e-05,
93
+ "loss": 1.4039306640625,
94
+ "step": 12
95
+ },
96
+ {
97
+ "epoch": 0.37142857142857144,
98
+ "grad_norm": 0.2551879286766052,
99
+ "learning_rate": 9.940426894506607e-05,
100
+ "loss": 1.32421875,
101
+ "step": 13
102
+ },
103
+ {
104
+ "epoch": 0.4,
105
+ "grad_norm": 0.35215961933135986,
106
+ "learning_rate": 9.92224291017886e-05,
107
+ "loss": 1.025390625,
108
+ "step": 14
109
+ },
110
+ {
111
+ "epoch": 0.42857142857142855,
112
+ "grad_norm": 0.2997990548610687,
113
+ "learning_rate": 9.901664203302126e-05,
114
+ "loss": 1.3216552734375,
115
+ "step": 15
116
+ },
117
+ {
118
+ "epoch": 0.45714285714285713,
119
+ "grad_norm": 0.25571194291114807,
120
+ "learning_rate": 9.878701917609207e-05,
121
+ "loss": 1.3499755859375,
122
+ "step": 16
123
+ },
124
+ {
125
+ "epoch": 0.4857142857142857,
126
+ "grad_norm": 0.34939518570899963,
127
+ "learning_rate": 9.853368487582887e-05,
128
+ "loss": 1.2916259765625,
129
+ "step": 17
130
+ },
131
+ {
132
+ "epoch": 0.5142857142857142,
133
+ "grad_norm": 0.17111709713935852,
134
+ "learning_rate": 9.825677631722435e-05,
135
+ "loss": 1.322265625,
136
+ "step": 18
137
+ },
138
+ {
139
+ "epoch": 0.5428571428571428,
140
+ "grad_norm": 0.15961293876171112,
141
+ "learning_rate": 9.795644345114795e-05,
142
+ "loss": 1.359375,
143
+ "step": 19
144
+ },
145
+ {
146
+ "epoch": 0.5714285714285714,
147
+ "grad_norm": 0.16286684572696686,
148
+ "learning_rate": 9.763284891314482e-05,
149
+ "loss": 1.212890625,
150
+ "step": 20
151
+ },
152
+ {
153
+ "epoch": 0.6,
154
+ "grad_norm": 0.15212465822696686,
155
+ "learning_rate": 9.728616793536588e-05,
156
+ "loss": 1.3544921875,
157
+ "step": 21
158
+ },
159
+ {
160
+ "epoch": 0.6285714285714286,
161
+ "grad_norm": 0.15721262991428375,
162
+ "learning_rate": 9.69165882516764e-05,
163
+ "loss": 1.29296875,
164
+ "step": 22
165
+ },
166
+ {
167
+ "epoch": 0.6571428571428571,
168
+ "grad_norm": 0.15810737013816833,
169
+ "learning_rate": 9.652430999599491e-05,
170
+ "loss": 1.2890625,
171
+ "step": 23
172
+ },
173
+ {
174
+ "epoch": 0.6857142857142857,
175
+ "grad_norm": 0.30460578203201294,
176
+ "learning_rate": 9.610954559391703e-05,
177
+ "loss": 1.294189453125,
178
+ "step": 24
179
+ },
180
+ {
181
+ "epoch": 0.7142857142857143,
182
+ "grad_norm": 0.17775993049144745,
183
+ "learning_rate": 9.567251964768343e-05,
184
+ "loss": 1.2939453125,
185
+ "step": 25
186
+ },
187
+ {
188
+ "epoch": 0.7428571428571429,
189
+ "grad_norm": 0.1571461409330368,
190
+ "learning_rate": 9.521346881455356e-05,
191
+ "loss": 1.2939453125,
192
+ "step": 26
193
+ },
194
+ {
195
+ "epoch": 0.7714285714285715,
196
+ "grad_norm": 0.11447877436876297,
197
+ "learning_rate": 9.473264167865173e-05,
198
+ "loss": 1.3115234375,
199
+ "step": 27
200
+ },
201
+ {
202
+ "epoch": 0.8,
203
+ "grad_norm": 0.08568654954433441,
204
+ "learning_rate": 9.423029861635431e-05,
205
+ "loss": 1.08123779296875,
206
+ "step": 28
207
+ },
208
+ {
209
+ "epoch": 0.8,
210
+ "eval_loss": 1.21484375,
211
+ "eval_runtime": 5.4015,
212
+ "eval_samples_per_second": 0.37,
213
+ "eval_steps_per_second": 0.185,
214
+ "step": 28
215
+ },
216
+ {
217
+ "epoch": 0.8285714285714286,
218
+ "grad_norm": 0.13479705154895782,
219
+ "learning_rate": 9.370671165529146e-05,
220
+ "loss": 1.2197265625,
221
+ "step": 29
222
+ },
223
+ {
224
+ "epoch": 0.8571428571428571,
225
+ "grad_norm": 0.13089671730995178,
226
+ "learning_rate": 9.316216432703917e-05,
227
+ "loss": 1.177734375,
228
+ "step": 30
229
+ },
230
+ {
231
+ "epoch": 0.8857142857142857,
232
+ "grad_norm": 0.16832147538661957,
233
+ "learning_rate": 9.259695151358214e-05,
234
+ "loss": 1.296875,
235
+ "step": 31
236
+ },
237
+ {
238
+ "epoch": 0.9142857142857143,
239
+ "grad_norm": 0.1125297099351883,
240
+ "learning_rate": 9.201137928762981e-05,
241
+ "loss": 1.28515625,
242
+ "step": 32
243
+ },
244
+ {
245
+ "epoch": 0.9428571428571428,
246
+ "grad_norm": 0.0942939817905426,
247
+ "learning_rate": 9.140576474687264e-05,
248
+ "loss": 1.24267578125,
249
+ "step": 33
250
+ },
251
+ {
252
+ "epoch": 0.9714285714285714,
253
+ "grad_norm": 0.11853406578302383,
254
+ "learning_rate": 9.078043584226815e-05,
255
+ "loss": 1.2470703125,
256
+ "step": 34
257
+ },
258
+ {
259
+ "epoch": 1.0,
260
+ "grad_norm": 0.8680324554443359,
261
+ "learning_rate": 9.013573120044967e-05,
262
+ "loss": 1.0714111328125,
263
+ "step": 35
264
+ },
265
+ {
266
+ "epoch": 1.0285714285714285,
267
+ "grad_norm": 0.0927174836397171,
268
+ "learning_rate": 8.947199994035401e-05,
269
+ "loss": 1.251953125,
270
+ "step": 36
271
+ },
272
+ {
273
+ "epoch": 1.0571428571428572,
274
+ "grad_norm": 0.10877764970064163,
275
+ "learning_rate": 8.878960148416748e-05,
276
+ "loss": 1.2294921875,
277
+ "step": 37
278
+ },
279
+ {
280
+ "epoch": 1.0857142857142856,
281
+ "grad_norm": 0.09218857437372208,
282
+ "learning_rate": 8.80889053626923e-05,
283
+ "loss": 1.220703125,
284
+ "step": 38
285
+ },
286
+ {
287
+ "epoch": 1.1142857142857143,
288
+ "grad_norm": 0.08481802046298981,
289
+ "learning_rate": 8.73702910152393e-05,
290
+ "loss": 1.19287109375,
291
+ "step": 39
292
+ },
293
+ {
294
+ "epoch": 1.1428571428571428,
295
+ "grad_norm": 0.08881324529647827,
296
+ "learning_rate": 8.663414758415479e-05,
297
+ "loss": 1.22021484375,
298
+ "step": 40
299
+ },
300
+ {
301
+ "epoch": 1.1714285714285715,
302
+ "grad_norm": 0.0994924008846283,
303
+ "learning_rate": 8.588087370409303e-05,
304
+ "loss": 1.271484375,
305
+ "step": 41
306
+ },
307
+ {
308
+ "epoch": 1.2,
309
+ "grad_norm": 0.09856198728084564,
310
+ "learning_rate": 8.511087728614862e-05,
311
+ "loss": 1.1400489807128906,
312
+ "step": 42
313
+ },
314
+ {
315
+ "epoch": 1.2285714285714286,
316
+ "grad_norm": 0.09289012849330902,
317
+ "learning_rate": 8.432457529696549e-05,
318
+ "loss": 1.2001953125,
319
+ "step": 43
320
+ },
321
+ {
322
+ "epoch": 1.2571428571428571,
323
+ "grad_norm": 0.10627222061157227,
324
+ "learning_rate": 8.352239353294196e-05,
325
+ "loss": 1.198577880859375,
326
+ "step": 44
327
+ },
328
+ {
329
+ "epoch": 1.2857142857142856,
330
+ "grad_norm": 0.15058334171772003,
331
+ "learning_rate": 8.270476638965462e-05,
332
+ "loss": 1.192840576171875,
333
+ "step": 45
334
+ },
335
+ {
336
+ "epoch": 1.3142857142857143,
337
+ "grad_norm": 0.09897656738758087,
338
+ "learning_rate": 8.187213662662538e-05,
339
+ "loss": 1.0825576782226562,
340
+ "step": 46
341
+ },
342
+ {
343
+ "epoch": 1.342857142857143,
344
+ "grad_norm": 0.09571877866983414,
345
+ "learning_rate": 8.102495512755939e-05,
346
+ "loss": 1.2222275733947754,
347
+ "step": 47
348
+ },
349
+ {
350
+ "epoch": 1.3714285714285714,
351
+ "grad_norm": 0.08556420356035233,
352
+ "learning_rate": 8.016368065618361e-05,
353
+ "loss": 1.16943359375,
354
+ "step": 48
355
+ },
356
+ {
357
+ "epoch": 1.4,
358
+ "grad_norm": 0.090641088783741,
359
+ "learning_rate": 7.928877960781808e-05,
360
+ "loss": 0.8678903579711914,
361
+ "step": 49
362
+ },
363
+ {
364
+ "epoch": 1.4285714285714286,
365
+ "grad_norm": 0.10227451473474503,
366
+ "learning_rate": 7.840072575681468e-05,
367
+ "loss": 1.1566047668457031,
368
+ "step": 50
369
+ },
370
+ {
371
+ "epoch": 1.457142857142857,
372
+ "grad_norm": 0.10105621814727783,
373
+ "learning_rate": 7.75e-05,
374
+ "loss": 1.198678970336914,
375
+ "step": 51
376
+ },
377
+ {
378
+ "epoch": 1.4857142857142858,
379
+ "grad_norm": 0.11792707443237305,
380
+ "learning_rate": 7.658709009626109e-05,
381
+ "loss": 1.1418228149414062,
382
+ "step": 52
383
+ },
384
+ {
385
+ "epoch": 1.5142857142857142,
386
+ "grad_norm": 0.096110500395298,
387
+ "learning_rate": 7.566249040241553e-05,
388
+ "loss": 1.1884765625,
389
+ "step": 53
390
+ },
391
+ {
392
+ "epoch": 1.5428571428571427,
393
+ "grad_norm": 0.10725893080234528,
394
+ "learning_rate": 7.472670160550849e-05,
395
+ "loss": 1.23095703125,
396
+ "step": 54
397
+ },
398
+ {
399
+ "epoch": 1.5714285714285714,
400
+ "grad_norm": 0.09755166620016098,
401
+ "learning_rate": 7.378023045168181e-05,
402
+ "loss": 1.1025390625,
403
+ "step": 55
404
+ },
405
+ {
406
+ "epoch": 1.6,
407
+ "grad_norm": 0.10026982426643372,
408
+ "learning_rate": 7.282358947176207e-05,
409
+ "loss": 1.22265625,
410
+ "step": 56
411
+ },
412
+ {
413
+ "epoch": 1.6,
414
+ "eval_loss": 1.140625,
415
+ "eval_runtime": 5.4002,
416
+ "eval_samples_per_second": 0.37,
417
+ "eval_steps_per_second": 0.185,
418
+ "step": 56
419
+ },
420
+ {
421
+ "epoch": 1.6285714285714286,
422
+ "grad_norm": 0.1034621149301529,
423
+ "learning_rate": 7.185729670371605e-05,
424
+ "loss": 1.1748046875,
425
+ "step": 57
426
+ },
427
+ {
428
+ "epoch": 1.657142857142857,
429
+ "grad_norm": 0.08397577702999115,
430
+ "learning_rate": 7.08818754121241e-05,
431
+ "loss": 1.1796875,
432
+ "step": 58
433
+ },
434
+ {
435
+ "epoch": 1.6857142857142857,
436
+ "grad_norm": 0.10789086669683456,
437
+ "learning_rate": 6.989785380482312e-05,
438
+ "loss": 1.1534786224365234,
439
+ "step": 59
440
+ },
441
+ {
442
+ "epoch": 1.7142857142857144,
443
+ "grad_norm": 0.10360898822546005,
444
+ "learning_rate": 6.890576474687263e-05,
445
+ "loss": 1.1875,
446
+ "step": 60
447
+ },
448
+ {
449
+ "epoch": 1.7428571428571429,
450
+ "grad_norm": 0.09336746484041214,
451
+ "learning_rate": 6.790614547199907e-05,
452
+ "loss": 1.1865234375,
453
+ "step": 61
454
+ },
455
+ {
456
+ "epoch": 1.7714285714285714,
457
+ "grad_norm": 0.1032928079366684,
458
+ "learning_rate": 6.689953729167411e-05,
459
+ "loss": 1.2080078125,
460
+ "step": 62
461
+ },
462
+ {
463
+ "epoch": 1.8,
464
+ "grad_norm": 0.0901721864938736,
465
+ "learning_rate": 6.588648530198504e-05,
466
+ "loss": 0.99957275390625,
467
+ "step": 63
468
+ },
469
+ {
470
+ "epoch": 1.8285714285714287,
471
+ "grad_norm": 0.08693459630012512,
472
+ "learning_rate": 6.486753808845564e-05,
473
+ "loss": 1.1259765625,
474
+ "step": 64
475
+ },
476
+ {
477
+ "epoch": 1.8571428571428572,
478
+ "grad_norm": 0.08747872710227966,
479
+ "learning_rate": 6.384324742897736e-05,
480
+ "loss": 1.09375,
481
+ "step": 65
482
+ },
483
+ {
484
+ "epoch": 1.8857142857142857,
485
+ "grad_norm": 0.10195285826921463,
486
+ "learning_rate": 6.281416799501188e-05,
487
+ "loss": 1.1982421875,
488
+ "step": 66
489
+ },
490
+ {
491
+ "epoch": 1.9142857142857141,
492
+ "grad_norm": 0.09723973274230957,
493
+ "learning_rate": 6.178085705122675e-05,
494
+ "loss": 1.1904296875,
495
+ "step": 67
496
+ },
497
+ {
498
+ "epoch": 1.9428571428571428,
499
+ "grad_norm": 0.0876585841178894,
500
+ "learning_rate": 6.0743874153726765e-05,
501
+ "loss": 1.16015625,
502
+ "step": 68
503
+ },
504
+ {
505
+ "epoch": 1.9714285714285715,
506
+ "grad_norm": 0.10089046508073807,
507
+ "learning_rate": 5.970378084704441e-05,
508
+ "loss": 1.16162109375,
509
+ "step": 69
510
+ },
511
+ {
512
+ "epoch": 2.0,
513
+ "grad_norm": 0.08220144361257553,
514
+ "learning_rate": 5.8661140360053624e-05,
515
+ "loss": 0.9820556640625,
516
+ "step": 70
517
+ },
518
+ {
519
+ "epoch": 2.0285714285714285,
520
+ "grad_norm": 0.09639836102724075,
521
+ "learning_rate": 5.761651730097142e-05,
522
+ "loss": 1.1689453125,
523
+ "step": 71
524
+ },
525
+ {
526
+ "epoch": 2.057142857142857,
527
+ "grad_norm": 0.08934585005044937,
528
+ "learning_rate": 5.657047735161256e-05,
529
+ "loss": 1.154296875,
530
+ "step": 72
531
+ },
532
+ {
533
+ "epoch": 2.085714285714286,
534
+ "grad_norm": 0.09140218794345856,
535
+ "learning_rate": 5.552358696106288e-05,
536
+ "loss": 1.1484375,
537
+ "step": 73
538
+ },
539
+ {
540
+ "epoch": 2.1142857142857143,
541
+ "grad_norm": 0.09645097702741623,
542
+ "learning_rate": 5.4476413038937144e-05,
543
+ "loss": 1.12060546875,
544
+ "step": 74
545
+ },
546
+ {
547
+ "epoch": 2.142857142857143,
548
+ "grad_norm": 0.09433187544345856,
549
+ "learning_rate": 5.342952264838747e-05,
550
+ "loss": 1.146484375,
551
+ "step": 75
552
+ },
553
+ {
554
+ "epoch": 2.1714285714285713,
555
+ "grad_norm": 0.10496731847524643,
556
+ "learning_rate": 5.23834826990286e-05,
557
+ "loss": 1.2001953125,
558
+ "step": 76
559
+ },
560
+ {
561
+ "epoch": 2.2,
562
+ "grad_norm": 0.10152551531791687,
563
+ "learning_rate": 5.1338859639946394e-05,
564
+ "loss": 1.070345163345337,
565
+ "step": 77
566
+ },
567
+ {
568
+ "epoch": 2.2285714285714286,
569
+ "grad_norm": 0.10176125913858414,
570
+ "learning_rate": 5.0296219152955604e-05,
571
+ "loss": 1.12890625,
572
+ "step": 78
573
+ },
574
+ {
575
+ "epoch": 2.257142857142857,
576
+ "grad_norm": 0.10605648159980774,
577
+ "learning_rate": 4.9256125846273246e-05,
578
+ "loss": 1.1309094429016113,
579
+ "step": 79
580
+ },
581
+ {
582
+ "epoch": 2.2857142857142856,
583
+ "grad_norm": 0.09611742198467255,
584
+ "learning_rate": 4.821914294877327e-05,
585
+ "loss": 1.1230978965759277,
586
+ "step": 80
587
+ },
588
+ {
589
+ "epoch": 2.314285714285714,
590
+ "grad_norm": 0.08913472294807434,
591
+ "learning_rate": 4.718583200498814e-05,
592
+ "loss": 1.0227570533752441,
593
+ "step": 81
594
+ },
595
+ {
596
+ "epoch": 2.342857142857143,
597
+ "grad_norm": 0.11219677329063416,
598
+ "learning_rate": 4.615675257102264e-05,
599
+ "loss": 1.1562614440917969,
600
+ "step": 82
601
+ },
602
+ {
603
+ "epoch": 2.3714285714285714,
604
+ "grad_norm": 0.10772853344678879,
605
+ "learning_rate": 4.513246191154434e-05,
606
+ "loss": 1.109375,
607
+ "step": 83
608
+ },
609
+ {
610
+ "epoch": 2.4,
611
+ "grad_norm": 0.08562427759170532,
612
+ "learning_rate": 4.411351469801496e-05,
613
+ "loss": 0.8189945220947266,
614
+ "step": 84
615
+ },
616
+ {
617
+ "epoch": 2.4,
618
+ "eval_loss": 1.109375,
619
+ "eval_runtime": 5.3939,
620
+ "eval_samples_per_second": 0.371,
621
+ "eval_steps_per_second": 0.185,
622
+ "step": 84
623
+ },
624
+ {
625
+ "epoch": 2.4285714285714284,
626
+ "grad_norm": 0.10952357202768326,
627
+ "learning_rate": 4.3100462708325916e-05,
628
+ "loss": 1.0928478240966797,
629
+ "step": 85
630
+ },
631
+ {
632
+ "epoch": 2.4571428571428573,
633
+ "grad_norm": 0.1026853621006012,
634
+ "learning_rate": 4.209385452800095e-05,
635
+ "loss": 1.1446080207824707,
636
+ "step": 86
637
+ },
638
+ {
639
+ "epoch": 2.4857142857142858,
640
+ "grad_norm": 0.10352090001106262,
641
+ "learning_rate": 4.109423525312738e-05,
642
+ "loss": 1.0831298828125,
643
+ "step": 87
644
+ },
645
+ {
646
+ "epoch": 2.5142857142857142,
647
+ "grad_norm": 0.10942506045103073,
648
+ "learning_rate": 4.010214619517689e-05,
649
+ "loss": 1.1318359375,
650
+ "step": 88
651
+ },
652
+ {
653
+ "epoch": 2.5428571428571427,
654
+ "grad_norm": 0.10975760221481323,
655
+ "learning_rate": 3.911812458787591e-05,
656
+ "loss": 1.177734375,
657
+ "step": 89
658
+ },
659
+ {
660
+ "epoch": 2.571428571428571,
661
+ "grad_norm": 0.09781485795974731,
662
+ "learning_rate": 3.814270329628396e-05,
663
+ "loss": 1.05078125,
664
+ "step": 90
665
+ },
666
+ {
667
+ "epoch": 2.6,
668
+ "grad_norm": 0.10182441025972366,
669
+ "learning_rate": 3.717641052823795e-05,
670
+ "loss": 1.1669921875,
671
+ "step": 91
672
+ },
673
+ {
674
+ "epoch": 2.6285714285714286,
675
+ "grad_norm": 0.11254462599754333,
676
+ "learning_rate": 3.62197695483182e-05,
677
+ "loss": 1.12255859375,
678
+ "step": 92
679
+ },
680
+ {
681
+ "epoch": 2.657142857142857,
682
+ "grad_norm": 0.09910311549901962,
683
+ "learning_rate": 3.5273298394491515e-05,
684
+ "loss": 1.1318359375,
685
+ "step": 93
686
+ },
687
+ {
688
+ "epoch": 2.685714285714286,
689
+ "grad_norm": 0.15467706322669983,
690
+ "learning_rate": 3.433750959758447e-05,
691
+ "loss": 1.1025604009628296,
692
+ "step": 94
693
+ },
694
+ {
695
+ "epoch": 2.7142857142857144,
696
+ "grad_norm": 0.09800928831100464,
697
+ "learning_rate": 3.3412909903738935e-05,
698
+ "loss": 1.1396484375,
699
+ "step": 95
700
+ },
701
+ {
702
+ "epoch": 2.742857142857143,
703
+ "grad_norm": 0.10627949237823486,
704
+ "learning_rate": 3.250000000000001e-05,
705
+ "loss": 1.1376953125,
706
+ "step": 96
707
+ },
708
+ {
709
+ "epoch": 2.7714285714285714,
710
+ "grad_norm": 0.1054811105132103,
711
+ "learning_rate": 3.159927424318531e-05,
712
+ "loss": 1.1630859375,
713
+ "step": 97
714
+ },
715
+ {
716
+ "epoch": 2.8,
717
+ "grad_norm": 0.09246952831745148,
718
+ "learning_rate": 3.071122039218194e-05,
719
+ "loss": 0.96343994140625,
720
+ "step": 98
721
+ },
722
+ {
723
+ "epoch": 2.8285714285714287,
724
+ "grad_norm": 0.09628032147884369,
725
+ "learning_rate": 2.98363193438164e-05,
726
+ "loss": 1.08544921875,
727
+ "step": 99
728
+ },
729
+ {
730
+ "epoch": 2.857142857142857,
731
+ "grad_norm": 0.09441180527210236,
732
+ "learning_rate": 2.897504487244061e-05,
733
+ "loss": 1.05615234375,
734
+ "step": 100
735
+ },
736
+ {
737
+ "epoch": 2.8857142857142857,
738
+ "grad_norm": 0.11135559529066086,
739
+ "learning_rate": 2.8127863373374635e-05,
740
+ "loss": 1.1552734375,
741
+ "step": 101
742
+ },
743
+ {
744
+ "epoch": 2.914285714285714,
745
+ "grad_norm": 0.1078437864780426,
746
+ "learning_rate": 2.729523361034538e-05,
747
+ "loss": 1.1484375,
748
+ "step": 102
749
+ },
750
+ {
751
+ "epoch": 2.942857142857143,
752
+ "grad_norm": 0.10249437391757965,
753
+ "learning_rate": 2.6477606467058035e-05,
754
+ "loss": 1.12060546875,
755
+ "step": 103
756
+ },
757
+ {
758
+ "epoch": 2.9714285714285715,
759
+ "grad_norm": 0.10100141167640686,
760
+ "learning_rate": 2.5675424703034524e-05,
761
+ "loss": 1.12109375,
762
+ "step": 104
763
+ },
764
+ {
765
+ "epoch": 3.0,
766
+ "grad_norm": 0.08678524196147919,
767
+ "learning_rate": 2.4889122713851394e-05,
768
+ "loss": 0.94921875,
769
+ "step": 105
770
+ },
771
+ {
772
+ "epoch": 3.0285714285714285,
773
+ "grad_norm": 0.10091595351696014,
774
+ "learning_rate": 2.4119126295906998e-05,
775
+ "loss": 1.12744140625,
776
+ "step": 106
777
+ },
778
+ {
779
+ "epoch": 3.057142857142857,
780
+ "grad_norm": 0.09988492727279663,
781
+ "learning_rate": 2.3365852415845223e-05,
782
+ "loss": 1.119140625,
783
+ "step": 107
784
+ },
785
+ {
786
+ "epoch": 3.085714285714286,
787
+ "grad_norm": 0.10282906144857407,
788
+ "learning_rate": 2.2629708984760708e-05,
789
+ "loss": 1.11279296875,
790
+ "step": 108
791
+ },
792
+ {
793
+ "epoch": 3.1142857142857143,
794
+ "grad_norm": 0.09999439865350723,
795
+ "learning_rate": 2.1911094637307716e-05,
796
+ "loss": 1.0869140625,
797
+ "step": 109
798
+ },
799
+ {
800
+ "epoch": 3.142857142857143,
801
+ "grad_norm": 0.1038166955113411,
802
+ "learning_rate": 2.121039851583254e-05,
803
+ "loss": 1.115234375,
804
+ "step": 110
805
+ },
806
+ {
807
+ "epoch": 3.1714285714285713,
808
+ "grad_norm": 0.10540751367807388,
809
+ "learning_rate": 2.0528000059645997e-05,
810
+ "loss": 1.1650390625,
811
+ "step": 111
812
+ },
813
+ {
814
+ "epoch": 3.2,
815
+ "grad_norm": 0.13892337679862976,
816
+ "learning_rate": 1.9864268799550338e-05,
817
+ "loss": 1.039089322090149,
818
+ "step": 112
819
+ },
820
+ {
821
+ "epoch": 3.2,
822
+ "eval_loss": 1.09765625,
823
+ "eval_runtime": 5.4123,
824
+ "eval_samples_per_second": 0.37,
825
+ "eval_steps_per_second": 0.185,
826
+ "step": 112
827
+ },
828
+ {
829
+ "epoch": 3.2285714285714286,
830
+ "grad_norm": 0.09683047980070114,
831
+ "learning_rate": 1.9219564157731846e-05,
832
+ "loss": 1.09814453125,
833
+ "step": 113
834
+ },
835
+ {
836
+ "epoch": 3.257142857142857,
837
+ "grad_norm": 0.10966203361749649,
838
+ "learning_rate": 1.8594235253127375e-05,
839
+ "loss": 1.097679853439331,
840
+ "step": 114
841
+ },
842
+ {
843
+ "epoch": 3.2857142857142856,
844
+ "grad_norm": 0.10792383551597595,
845
+ "learning_rate": 1.7988620712370196e-05,
846
+ "loss": 1.093778371810913,
847
+ "step": 115
848
+ },
849
+ {
850
+ "epoch": 3.314285714285714,
851
+ "grad_norm": 0.09936859458684921,
852
+ "learning_rate": 1.740304848641787e-05,
853
+ "loss": 0.99672532081604,
854
+ "step": 116
855
+ },
856
+ {
857
+ "epoch": 3.342857142857143,
858
+ "grad_norm": 0.1135304793715477,
859
+ "learning_rate": 1.6837835672960835e-05,
860
+ "loss": 1.1254997253417969,
861
+ "step": 117
862
+ },
863
+ {
864
+ "epoch": 3.3714285714285714,
865
+ "grad_norm": 0.10551383346319199,
866
+ "learning_rate": 1.6293288344708568e-05,
867
+ "loss": 1.0830078125,
868
+ "step": 118
869
+ },
870
+ {
871
+ "epoch": 3.4,
872
+ "grad_norm": 0.09492458403110504,
873
+ "learning_rate": 1.57697013836457e-05,
874
+ "loss": 0.7991197109222412,
875
+ "step": 119
876
+ },
877
+ {
878
+ "epoch": 3.4285714285714284,
879
+ "grad_norm": 0.10766921192407608,
880
+ "learning_rate": 1.526735832134829e-05,
881
+ "loss": 1.0654666423797607,
882
+ "step": 120
883
+ },
884
+ {
885
+ "epoch": 3.4571428571428573,
886
+ "grad_norm": 0.1085314080119133,
887
+ "learning_rate": 1.4786531185446454e-05,
888
+ "loss": 1.1201512813568115,
889
+ "step": 121
890
+ },
891
+ {
892
+ "epoch": 3.4857142857142858,
893
+ "grad_norm": 0.11182897537946701,
894
+ "learning_rate": 1.432748035231658e-05,
895
+ "loss": 1.0567700862884521,
896
+ "step": 122
897
+ },
898
+ {
899
+ "epoch": 3.5142857142857142,
900
+ "grad_norm": 0.10476535558700562,
901
+ "learning_rate": 1.389045440608296e-05,
902
+ "loss": 1.10693359375,
903
+ "step": 123
904
+ },
905
+ {
906
+ "epoch": 3.5428571428571427,
907
+ "grad_norm": 0.10022886097431183,
908
+ "learning_rate": 1.3475690004005098e-05,
909
+ "loss": 1.15234375,
910
+ "step": 124
911
+ },
912
+ {
913
+ "epoch": 3.571428571428571,
914
+ "grad_norm": 0.09982618689537048,
915
+ "learning_rate": 1.3083411748323591e-05,
916
+ "loss": 1.029296875,
917
+ "step": 125
918
+ },
919
+ {
920
+ "epoch": 3.6,
921
+ "grad_norm": 0.10781899839639664,
922
+ "learning_rate": 1.2713832064634126e-05,
923
+ "loss": 1.14453125,
924
+ "step": 126
925
+ },
926
+ {
927
+ "epoch": 3.6285714285714286,
928
+ "grad_norm": 0.10565479844808578,
929
+ "learning_rate": 1.2367151086855187e-05,
930
+ "loss": 1.10302734375,
931
+ "step": 127
932
+ },
933
+ {
934
+ "epoch": 3.657142857142857,
935
+ "grad_norm": 0.09674814343452454,
936
+ "learning_rate": 1.2043556548852065e-05,
937
+ "loss": 1.11083984375,
938
+ "step": 128
939
+ },
940
+ {
941
+ "epoch": 3.685714285714286,
942
+ "grad_norm": 0.10641477257013321,
943
+ "learning_rate": 1.174322368277565e-05,
944
+ "loss": 1.0820510387420654,
945
+ "step": 129
946
+ },
947
+ {
948
+ "epoch": 3.7142857142857144,
949
+ "grad_norm": 0.09659207612276077,
950
+ "learning_rate": 1.146631512417113e-05,
951
+ "loss": 1.11962890625,
952
+ "step": 130
953
+ },
954
+ {
955
+ "epoch": 3.742857142857143,
956
+ "grad_norm": 0.10608964413404465,
957
+ "learning_rate": 1.1212980823907931e-05,
958
+ "loss": 1.12060546875,
959
+ "step": 131
960
+ },
961
+ {
962
+ "epoch": 3.7714285714285714,
963
+ "grad_norm": 0.10524872690439224,
964
+ "learning_rate": 1.0983357966978745e-05,
965
+ "loss": 1.14306640625,
966
+ "step": 132
967
+ },
968
+ {
969
+ "epoch": 3.8,
970
+ "grad_norm": 0.09818793088197708,
971
+ "learning_rate": 1.0777570898211406e-05,
972
+ "loss": 0.94818115234375,
973
+ "step": 133
974
+ },
975
+ {
976
+ "epoch": 3.8285714285714287,
977
+ "grad_norm": 0.09379758685827255,
978
+ "learning_rate": 1.0595731054933936e-05,
979
+ "loss": 1.06787109375,
980
+ "step": 134
981
+ },
982
+ {
983
+ "epoch": 3.857142857142857,
984
+ "grad_norm": 0.09733222424983978,
985
+ "learning_rate": 1.0437936906629336e-05,
986
+ "loss": 1.0400390625,
987
+ "step": 135
988
+ },
989
+ {
990
+ "epoch": 3.8857142857142857,
991
+ "grad_norm": 0.10898660868406296,
992
+ "learning_rate": 1.0304273901612566e-05,
993
+ "loss": 1.13818359375,
994
+ "step": 136
995
+ },
996
+ {
997
+ "epoch": 3.914285714285714,
998
+ "grad_norm": 0.11681357771158218,
999
+ "learning_rate": 1.0194814420758805e-05,
1000
+ "loss": 1.1318359375,
1001
+ "step": 137
1002
+ },
1003
+ {
1004
+ "epoch": 3.942857142857143,
1005
+ "grad_norm": 0.10775525122880936,
1006
+ "learning_rate": 1.0109617738307912e-05,
1007
+ "loss": 1.10498046875,
1008
+ "step": 138
1009
+ },
1010
+ {
1011
+ "epoch": 3.9714285714285715,
1012
+ "grad_norm": 0.10294745117425919,
1013
+ "learning_rate": 1.0048729989766395e-05,
1014
+ "loss": 1.1083984375,
1015
+ "step": 139
1016
+ },
1017
+ {
1018
+ "epoch": 4.0,
1019
+ "grad_norm": 0.08989562839269638,
1020
+ "learning_rate": 1.0012184146924224e-05,
1021
+ "loss": 0.934478759765625,
1022
+ "step": 140
1023
+ },
1024
+ {
1025
+ "epoch": 4.0,
1026
+ "eval_loss": 1.08984375,
1027
+ "eval_runtime": 5.3982,
1028
+ "eval_samples_per_second": 0.37,
1029
+ "eval_steps_per_second": 0.185,
1030
+ "step": 140
1031
+ }
1032
+ ],
1033
+ "logging_steps": 1.0,
1034
+ "max_steps": 140,
1035
+ "num_input_tokens_seen": 0,
1036
+ "num_train_epochs": 4,
1037
+ "save_steps": 0,
1038
+ "stateful_callbacks": {
1039
+ "TrainerControl": {
1040
+ "args": {
1041
+ "should_epoch_stop": false,
1042
+ "should_evaluate": false,
1043
+ "should_log": false,
1044
+ "should_save": true,
1045
+ "should_training_stop": true
1046
+ },
1047
+ "attributes": {}
1048
+ }
1049
+ },
1050
+ "total_flos": 1.2160898563718513e+19,
1051
+ "train_batch_size": 1,
1052
+ "trial_name": null,
1053
+ "trial_params": null
1054
+ }
training-metrics.png ADDED
win-rates.png ADDED