LucaFrat commited on
Commit
6c0ddc4
·
verified ·
1 Parent(s): 61debb7

Delete files checkpoint-4* with huggingface_hub

Browse files
checkpoint-4000/config.json DELETED
@@ -1,70 +0,0 @@
1
- {
2
- "action_horizon": 50,
3
- "add_pos_embed": true,
4
- "apply_sincos_state_encoding": true,
5
- "architectures": [
6
- "Gr00tN1d6"
7
- ],
8
- "attn_dropout": 0.2,
9
- "attn_implementation": null,
10
- "backbone_embedding_dim": 2048,
11
- "backbone_model_type": "eagle",
12
- "backbone_trainable_params_fp32": true,
13
- "collator_overwrite_image_inputs": false,
14
- "color_jitter_params": {
15
- "brightness": 0.1,
16
- "contrast": 0.1,
17
- "hue": 0.1,
18
- "saturation": 0.1
19
- },
20
- "crop_fraction": 0.95,
21
- "diffusion_model_cfg": {
22
- "attention_head_dim": 48,
23
- "dropout": 0.2,
24
- "final_dropout": true,
25
- "interleave_self_attention": true,
26
- "norm_type": "ada_norm",
27
- "num_attention_heads": 32,
28
- "num_layers": 32,
29
- "output_dim": 1024,
30
- "positional_embeddings": null
31
- },
32
- "eagle_collator": true,
33
- "formalize_language": true,
34
- "gemma_collator": false,
35
- "hidden_size": 1024,
36
- "image_crop_size": null,
37
- "image_target_size": null,
38
- "input_embedding_dim": 1536,
39
- "load_bf16": true,
40
- "max_action_dim": 128,
41
- "max_num_embodiments": 32,
42
- "max_seq_len": 1024,
43
- "max_state_dim": 128,
44
- "model_dtype": "bfloat16",
45
- "model_name": "nvidia/Eagle-Block2A-2B-v2",
46
- "model_type": "Gr00tN1d6",
47
- "noise_beta_alpha": 1.5,
48
- "noise_beta_beta": 1.0,
49
- "noise_s": 0.999,
50
- "num_inference_timesteps": 4,
51
- "num_timestep_buckets": 1000,
52
- "random_rotation_angle": null,
53
- "reproject_vision": false,
54
- "select_layer": 16,
55
- "shortest_image_edge": 256,
56
- "state_dropout_prob": 0.0,
57
- "torch_dtype": "bfloat16",
58
- "transformers_version": "4.51.3",
59
- "tune_diffusion_model": true,
60
- "tune_llm": false,
61
- "tune_projector": true,
62
- "tune_top_llm_layers": 4,
63
- "tune_visual": false,
64
- "tune_vlln": true,
65
- "use_albumentations_transforms": true,
66
- "use_alternate_vl_dit": true,
67
- "use_flash_attention": true,
68
- "use_relative_action": true,
69
- "use_vlln": true
70
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-4000/embodiment_id.json DELETED
@@ -1,11 +0,0 @@
1
- {
2
- "robocasa_panda_omron": 13,
3
- "gr1": 20,
4
- "behavior_r1_pro": 24,
5
- "unitree_g1": 8,
6
- "oxe_google": 0,
7
- "oxe_widowx": 1,
8
- "libero_panda": 2,
9
- "oxe_droid": 16,
10
- "new_embodiment": 10
11
- }
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-4000/experiment_cfg/conf.yaml DELETED
@@ -1,252 +0,0 @@
1
- load_config_path: null
2
- model:
3
- model_type: Gr00tN1d6
4
- model_dtype: bfloat16
5
- model_name: nvidia/Eagle-Block2A-2B-v2
6
- backbone_model_type: eagle
7
- model_revision: null
8
- tune_top_llm_layers: 4
9
- backbone_embedding_dim: 2048
10
- tune_llm: false
11
- tune_visual: false
12
- select_layer: 16
13
- reproject_vision: false
14
- use_flash_attention: true
15
- load_bf16: false
16
- collator_overwrite_image_inputs: false
17
- eagle_collator: true
18
- backbone_trainable_params_fp32: true
19
- image_crop_size: null
20
- image_target_size: null
21
- shortest_image_edge: 256
22
- crop_fraction: 0.95
23
- random_rotation_angle: null
24
- color_jitter_params:
25
- brightness: 0.3
26
- contrast: 0.4
27
- saturation: 0.5
28
- hue: 0.08
29
- use_albumentations_transforms: true
30
- formalize_language: true
31
- apply_sincos_state_encoding: false
32
- use_relative_action: true
33
- max_state_dim: 29
34
- max_action_dim: 29
35
- action_horizon: 16
36
- hidden_size: 1024
37
- input_embedding_dim: 1536
38
- add_pos_embed: true
39
- attn_dropout: 0.2
40
- use_vlln: true
41
- max_seq_len: 1024
42
- use_alternate_vl_dit: true
43
- attend_text_every_n_blocks: 2
44
- diffusion_model_cfg:
45
- positional_embeddings: null
46
- num_layers: 32
47
- num_attention_heads: 32
48
- attention_head_dim: 48
49
- norm_type: ada_norm
50
- dropout: 0.2
51
- final_dropout: true
52
- output_dim: 1024
53
- interleave_self_attention: true
54
- num_inference_timesteps: 4
55
- noise_beta_alpha: 1.5
56
- noise_beta_beta: 1.0
57
- noise_s: 0.999
58
- num_timestep_buckets: 1000
59
- tune_projector: true
60
- tune_diffusion_model: true
61
- tune_vlln: true
62
- state_dropout_prob: 0.0
63
- state_additive_noise_scale: 0.0
64
- max_num_embodiments: 32
65
- data:
66
- datasets:
67
- - dataset_paths:
68
- - /data/PnPCubeLine
69
- embodiment_tag: unitree_g1
70
- mix_ratio: 1.0
71
- dataset_type: physical_embodiment
72
- val_dataset_path: null
73
- modality_configs:
74
- unitree_g1:
75
- video:
76
- delta_indices:
77
- - 0
78
- modality_keys:
79
- - ego_view
80
- sin_cos_embedding_keys: null
81
- mean_std_embedding_keys: null
82
- action_configs: null
83
- state:
84
- delta_indices:
85
- - 0
86
- modality_keys:
87
- - left_leg
88
- - right_leg
89
- - waist
90
- - left_arm
91
- - right_arm
92
- - left_hand
93
- - right_hand
94
- sin_cos_embedding_keys: null
95
- mean_std_embedding_keys: null
96
- action_configs: null
97
- action:
98
- delta_indices:
99
- - 0
100
- - 1
101
- - 2
102
- - 3
103
- - 4
104
- - 5
105
- - 6
106
- - 7
107
- - 8
108
- - 9
109
- - 10
110
- - 11
111
- - 12
112
- - 13
113
- - 14
114
- - 15
115
- - 16
116
- - 17
117
- - 18
118
- - 19
119
- - 20
120
- - 21
121
- - 22
122
- - 23
123
- - 24
124
- - 25
125
- - 26
126
- - 27
127
- - 28
128
- - 29
129
- modality_keys:
130
- - left_arm
131
- - right_arm
132
- - left_hand
133
- - right_hand
134
- - waist
135
- - base_height_command
136
- - navigate_command
137
- sin_cos_embedding_keys: null
138
- mean_std_embedding_keys: null
139
- action_configs:
140
- - rep: RELATIVE
141
- type: NON_EEF
142
- format: DEFAULT
143
- state_key: null
144
- - rep: RELATIVE
145
- type: NON_EEF
146
- format: DEFAULT
147
- state_key: null
148
- - rep: ABSOLUTE
149
- type: NON_EEF
150
- format: DEFAULT
151
- state_key: null
152
- - rep: ABSOLUTE
153
- type: NON_EEF
154
- format: DEFAULT
155
- state_key: null
156
- - rep: ABSOLUTE
157
- type: NON_EEF
158
- format: DEFAULT
159
- state_key: null
160
- - rep: ABSOLUTE
161
- type: NON_EEF
162
- format: DEFAULT
163
- state_key: null
164
- - rep: ABSOLUTE
165
- type: NON_EEF
166
- format: DEFAULT
167
- state_key: null
168
- language:
169
- delta_indices:
170
- - 0
171
- modality_keys:
172
- - annotation.human.task_description
173
- sin_cos_embedding_keys: null
174
- mean_std_embedding_keys: null
175
- action_configs: null
176
- download_cache: false
177
- shard_size: 1024
178
- episode_sampling_rate: 0.1
179
- num_shards_per_epoch: 100000
180
- override_pretraining_statistics: false
181
- mode: single_turn
182
- random_chop: 0.0
183
- mock_dataset_mode: false
184
- shuffle: true
185
- seed: 42
186
- multiprocessing_context: fork
187
- allow_padding: false
188
- subsample_ratio: 1.0
189
- image_crop_size:
190
- - 244
191
- - 244
192
- image_target_size:
193
- - 224
194
- - 224
195
- video_backend: torchcodec
196
- training:
197
- output_dir: /checkpoints/g1_finetune
198
- experiment_name: null
199
- max_steps: 6000
200
- global_batch_size: 512
201
- batch_size: null
202
- gradient_accumulation_steps: 1
203
- learning_rate: 0.0001
204
- lr_scheduler_type: cosine
205
- weight_decay: 1.0e-05
206
- warmup_ratio: 0.05
207
- warmup_steps: 0
208
- max_grad_norm: 1.0
209
- optim: adamw_torch
210
- start_from_checkpoint: nvidia/GR00T-N1.6-3B
211
- tf32: true
212
- fp16: false
213
- bf16: true
214
- eval_bf16: true
215
- logging_steps: 10
216
- save_steps: 2000
217
- save_total_limit: 2
218
- save_vl_model: false
219
- upload_checkpoints: false
220
- upload_every: 1000
221
- upload_last_n_checkpoints: 5
222
- max_concurrent_uploads: 2
223
- eval_strategy: 'no'
224
- eval_steps: 500
225
- eval_set_split_ratio: 0.1
226
- eval_batch_size: 2
227
- save_best_eval_metric_name: ''
228
- save_best_eval_metric_greater_is_better: true
229
- deepspeed_stage: 2
230
- gradient_checkpointing: false
231
- transformers_trust_remote_code: true
232
- transformers_local_files_only: false
233
- transformers_cache_dir: null
234
- transformers_access_token: null
235
- use_ddp: false
236
- ddp_bucket_cap_mb: 100
237
- num_gpus: 4
238
- dataloader_num_workers: 6
239
- remove_unused_columns: false
240
- use_wandb: false
241
- wandb_project: finetune-gr00t-n1d6
242
- enable_profiling: false
243
- max_retries: 3
244
- assert_loss_less_than: null
245
- add_rl_callback: false
246
- enable_open_loop_eval: false
247
- open_loop_eval_traj_ids:
248
- - 0
249
- open_loop_eval_steps_per_traj: 100
250
- open_loop_eval_plot_indices: null
251
- max_steps: 6000
252
- save_steps: 2000
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-4000/experiment_cfg/config.yaml DELETED
@@ -1,291 +0,0 @@
1
- !!python/object:gr00t.configs.base_config.Config
2
- data: !!python/object:gr00t.configs.data.data_config.DataConfig
3
- allow_padding: false
4
- datasets:
5
- - !!python/object:gr00t.configs.data.data_config.SingleDatasetConfig
6
- dataset_paths:
7
- - /data/PnPCubeLine
8
- dataset_type: physical_embodiment
9
- embodiment_tag: unitree_g1
10
- mix_ratio: 1.0
11
- val_dataset_path: null
12
- download_cache: false
13
- episode_sampling_rate: 0.1
14
- image_crop_size:
15
- - 244
16
- - 244
17
- image_target_size:
18
- - 224
19
- - 224
20
- mock_dataset_mode: false
21
- modality_configs:
22
- unitree_g1:
23
- action: !!python/object:gr00t.data.types.ModalityConfig
24
- action_configs:
25
- - !!python/object:gr00t.data.types.ActionConfig
26
- format: &id001 !!python/object/apply:gr00t.data.types.ActionFormat
27
- - default
28
- rep: &id002 !!python/object/apply:gr00t.data.types.ActionRepresentation
29
- - relative
30
- state_key: null
31
- type: &id003 !!python/object/apply:gr00t.data.types.ActionType
32
- - non_eef
33
- - !!python/object:gr00t.data.types.ActionConfig
34
- format: *id001
35
- rep: *id002
36
- state_key: null
37
- type: *id003
38
- - !!python/object:gr00t.data.types.ActionConfig
39
- format: *id001
40
- rep: &id004 !!python/object/apply:gr00t.data.types.ActionRepresentation
41
- - absolute
42
- state_key: null
43
- type: *id003
44
- - !!python/object:gr00t.data.types.ActionConfig
45
- format: *id001
46
- rep: *id004
47
- state_key: null
48
- type: *id003
49
- - !!python/object:gr00t.data.types.ActionConfig
50
- format: *id001
51
- rep: *id004
52
- state_key: null
53
- type: *id003
54
- - !!python/object:gr00t.data.types.ActionConfig
55
- format: *id001
56
- rep: *id004
57
- state_key: null
58
- type: *id003
59
- - !!python/object:gr00t.data.types.ActionConfig
60
- format: *id001
61
- rep: *id004
62
- state_key: null
63
- type: *id003
64
- delta_indices:
65
- - 0
66
- - 1
67
- - 2
68
- - 3
69
- - 4
70
- - 5
71
- - 6
72
- - 7
73
- - 8
74
- - 9
75
- - 10
76
- - 11
77
- - 12
78
- - 13
79
- - 14
80
- - 15
81
- - 16
82
- - 17
83
- - 18
84
- - 19
85
- - 20
86
- - 21
87
- - 22
88
- - 23
89
- - 24
90
- - 25
91
- - 26
92
- - 27
93
- - 28
94
- - 29
95
- mean_std_embedding_keys: null
96
- modality_keys:
97
- - left_arm
98
- - right_arm
99
- - left_hand
100
- - right_hand
101
- - waist
102
- - base_height_command
103
- - navigate_command
104
- sin_cos_embedding_keys: null
105
- language: !!python/object:gr00t.data.types.ModalityConfig
106
- action_configs: null
107
- delta_indices:
108
- - 0
109
- mean_std_embedding_keys: null
110
- modality_keys:
111
- - annotation.human.task_description
112
- sin_cos_embedding_keys: null
113
- state: !!python/object:gr00t.data.types.ModalityConfig
114
- action_configs: null
115
- delta_indices:
116
- - 0
117
- mean_std_embedding_keys: null
118
- modality_keys:
119
- - left_leg
120
- - right_leg
121
- - waist
122
- - left_arm
123
- - right_arm
124
- - left_hand
125
- - right_hand
126
- sin_cos_embedding_keys: null
127
- video: !!python/object:gr00t.data.types.ModalityConfig
128
- action_configs: null
129
- delta_indices:
130
- - 0
131
- mean_std_embedding_keys: null
132
- modality_keys:
133
- - ego_view
134
- sin_cos_embedding_keys: null
135
- mode: single_turn
136
- multiprocessing_context: fork
137
- num_shards_per_epoch: 100000
138
- override_pretraining_statistics: false
139
- random_chop: 0.0
140
- seed: 42
141
- shard_size: 1024
142
- shuffle: true
143
- subsample_ratio: 1.0
144
- video_backend: torchcodec
145
- load_config_path: null
146
- model: !!python/object:gr00t.configs.model.gr00t_n1d6.Gr00tN1d6Config
147
- _attn_implementation_autoset: false
148
- _attn_implementation_internal: null
149
- _commit_hash: null
150
- _name_or_path: ''
151
- add_cross_attention: false
152
- architectures: null
153
- backbone_model_type: eagle
154
- backbone_trainable_params_fp32: true
155
- bad_words_ids: null
156
- begin_suppress_tokens: null
157
- bos_token_id: null
158
- chunk_size_feed_forward: 0
159
- color_jitter_params:
160
- brightness: 0.3
161
- contrast: 0.4
162
- hue: 0.08
163
- saturation: 0.5
164
- cross_attention_hidden_size: null
165
- decoder_start_token_id: null
166
- diffusion_model_cfg:
167
- attention_head_dim: 48
168
- dropout: 0.2
169
- final_dropout: true
170
- interleave_self_attention: true
171
- norm_type: ada_norm
172
- num_attention_heads: 32
173
- num_layers: 32
174
- output_dim: 1024
175
- positional_embeddings: null
176
- diversity_penalty: 0.0
177
- do_sample: false
178
- eagle_collator: true
179
- early_stopping: false
180
- encoder_no_repeat_ngram_size: 0
181
- eos_token_id: null
182
- exponential_decay_length_penalty: null
183
- finetuning_task: null
184
- forced_bos_token_id: null
185
- forced_eos_token_id: null
186
- id2label:
187
- 0: LABEL_0
188
- 1: LABEL_1
189
- is_decoder: false
190
- is_encoder_decoder: false
191
- label2id:
192
- LABEL_0: 0
193
- LABEL_1: 1
194
- length_penalty: 1.0
195
- load_bf16: false
196
- max_length: 20
197
- min_length: 0
198
- model_name: nvidia/Eagle-Block2A-2B-v2
199
- no_repeat_ngram_size: 0
200
- num_beam_groups: 1
201
- num_beams: 1
202
- num_return_sequences: 1
203
- output_attentions: false
204
- output_hidden_states: false
205
- output_scores: false
206
- pad_token_id: null
207
- prefix: null
208
- problem_type: null
209
- pruned_heads: {}
210
- random_rotation_angle: null
211
- remove_invalid_values: false
212
- repetition_penalty: 1.0
213
- reproject_vision: false
214
- return_dict: true
215
- return_dict_in_generate: false
216
- sep_token_id: null
217
- state_dropout_prob: 0.0
218
- suppress_tokens: null
219
- task_specific_params: null
220
- temperature: 1.0
221
- tf_legacy_loss: false
222
- tie_encoder_decoder: false
223
- tie_word_embeddings: true
224
- tokenizer_class: null
225
- top_k: 50
226
- top_p: 1.0
227
- torch_dtype: null
228
- torchscript: false
229
- transformers_version: null
230
- tune_diffusion_model: true
231
- tune_llm: false
232
- tune_projector: true
233
- tune_visual: false
234
- typical_p: 1.0
235
- use_bfloat16: false
236
- use_relative_action: true
237
- training: !!python/object:gr00t.configs.training.training_config.TrainingConfig
238
- add_rl_callback: false
239
- assert_loss_less_than: null
240
- batch_size: null
241
- bf16: true
242
- dataloader_num_workers: 6
243
- ddp_bucket_cap_mb: 100
244
- deepspeed_stage: 2
245
- enable_open_loop_eval: false
246
- enable_profiling: false
247
- eval_batch_size: 2
248
- eval_bf16: true
249
- eval_set_split_ratio: 0.1
250
- eval_steps: 500
251
- eval_strategy: 'no'
252
- experiment_name: null
253
- fp16: false
254
- global_batch_size: 512
255
- gradient_accumulation_steps: 1
256
- gradient_checkpointing: false
257
- learning_rate: 0.0001
258
- logging_steps: 10
259
- lr_scheduler_type: cosine
260
- max_concurrent_uploads: 2
261
- max_grad_norm: 1.0
262
- max_retries: 3
263
- max_steps: 6000
264
- num_gpus: 4
265
- open_loop_eval_plot_indices: null
266
- open_loop_eval_steps_per_traj: 100
267
- open_loop_eval_traj_ids:
268
- - 0
269
- optim: adamw_torch
270
- output_dir: /checkpoints/g1_finetune
271
- remove_unused_columns: false
272
- save_best_eval_metric_greater_is_better: true
273
- save_best_eval_metric_name: ''
274
- save_steps: 2000
275
- save_total_limit: 2
276
- save_vl_model: false
277
- start_from_checkpoint: nvidia/GR00T-N1.6-3B
278
- tf32: true
279
- transformers_access_token: null
280
- transformers_cache_dir: null
281
- transformers_local_files_only: false
282
- transformers_trust_remote_code: true
283
- upload_checkpoints: false
284
- upload_every: 1000
285
- upload_last_n_checkpoints: 5
286
- use_ddp: false
287
- use_wandb: false
288
- wandb_project: finetune-gr00t-n1d6
289
- warmup_ratio: 0.05
290
- warmup_steps: 0
291
- weight_decay: 1.0e-05
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-4000/experiment_cfg/dataset_statistics.json DELETED
The diff for this file is too large to render. See raw diff
 
checkpoint-4000/experiment_cfg/final_model_config.json DELETED
@@ -1,53 +0,0 @@
1
- {
2
- "model_type": "Gr00tN1d6",
3
- "model_dtype": "bfloat16",
4
- "model_name": "nvidia/Eagle-Block2A-2B-v2",
5
- "backbone_model_type": "eagle",
6
- "model_revision": null,
7
- "tune_top_llm_layers": 4,
8
- "backbone_embedding_dim": 2048,
9
- "tune_llm": false,
10
- "tune_visual": false,
11
- "select_layer": 16,
12
- "reproject_vision": false,
13
- "use_flash_attention": true,
14
- "load_bf16": true,
15
- "collator_overwrite_image_inputs": false,
16
- "eagle_collator": true,
17
- "backbone_trainable_params_fp32": true,
18
- "apply_sincos_state_encoding": true,
19
- "use_relative_action": true,
20
- "max_state_dim": 128,
21
- "max_action_dim": 128,
22
- "action_horizon": 50,
23
- "hidden_size": 1024,
24
- "input_embedding_dim": 1536,
25
- "add_pos_embed": true,
26
- "attn_dropout": 0.2,
27
- "use_vlln": true,
28
- "max_seq_len": 1024,
29
- "use_alternate_vl_dit": true,
30
- "attend_text_every_n_blocks": 2,
31
- "diffusion_model_cfg": {
32
- "attention_head_dim": 48,
33
- "dropout": 0.2,
34
- "final_dropout": true,
35
- "interleave_self_attention": true,
36
- "norm_type": "ada_norm",
37
- "num_attention_heads": 32,
38
- "num_layers": 32,
39
- "output_dim": 1024,
40
- "positional_embeddings": null
41
- },
42
- "num_inference_timesteps": 4,
43
- "noise_beta_alpha": 1.5,
44
- "noise_beta_beta": 1.0,
45
- "noise_s": 0.999,
46
- "num_timestep_buckets": 1000,
47
- "tune_projector": true,
48
- "tune_diffusion_model": true,
49
- "tune_vlln": true,
50
- "state_dropout_prob": 0.0,
51
- "state_additive_noise_scale": 0.0,
52
- "max_num_embodiments": 32
53
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-4000/experiment_cfg/final_processor_config.json DELETED
The diff for this file is too large to render. See raw diff
 
checkpoint-4000/global_step4000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:3119b9ba23e36428ae55e9e1d0c3900eb58683bbae2020edd8e8f658b9c45b2e
3
- size 4859923697
 
 
 
 
checkpoint-4000/global_step4000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:b3be7efbf34e0c90d79fa285371ed108c7f6d8b2113ad5e0f58be74287d943b0
3
- size 4859923761
 
 
 
 
checkpoint-4000/global_step4000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:b74ef7944e50282f18551511fab67660dc5962c0c57125cf1b88896ae639787f
3
- size 4859923569
 
 
 
 
checkpoint-4000/global_step4000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:90c5e1afc967bfb361e0d9f2fb9a42945690fad8e48269432ac62d35b7729589
3
- size 4859917873
 
 
 
 
checkpoint-4000/global_step4000/mp_rank_00_model_states.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:0167eeda5fa81617a214708b8358332c5beb9ddccd03d63194b62acc7ad08f0f
3
- size 9907202435
 
 
 
 
checkpoint-4000/latest DELETED
@@ -1 +0,0 @@
1
- global_step4000
 
 
checkpoint-4000/model-00001-of-00002.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:7b21864df58a76a4372a0049731ae1c343137e01644798e208523907ede4c8cc
3
- size 4991091456
 
 
 
 
checkpoint-4000/model-00002-of-00002.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:5a01e9468a42fcf825c860edcff7be66f5bde3924216e42b3a99ac9b122d63d3
3
- size 1582283096
 
 
 
 
checkpoint-4000/model.safetensors.index.json DELETED
The diff for this file is too large to render. See raw diff
 
checkpoint-4000/processor_config.json DELETED
@@ -1,508 +0,0 @@
1
- {
2
- "processor_class": "Gr00tN1d6Processor",
3
- "processor_kwargs": {
4
- "modality_configs": {
5
- "behavior_r1_pro": {
6
- "video": {
7
- "delta_indices": [
8
- 0
9
- ],
10
- "modality_keys": [
11
- "observation.images.rgb.head_256_256",
12
- "observation.images.rgb.left_wrist_256_256",
13
- "observation.images.rgb.right_wrist_256_256"
14
- ],
15
- "sin_cos_embedding_keys": null,
16
- "mean_std_embedding_keys": null,
17
- "action_configs": null
18
- },
19
- "state": {
20
- "delta_indices": [
21
- 0
22
- ],
23
- "modality_keys": [
24
- "robot_pos",
25
- "robot_ori_cos",
26
- "robot_ori_sin",
27
- "robot_2d_ori",
28
- "robot_2d_ori_cos",
29
- "robot_2d_ori_sin",
30
- "robot_lin_vel",
31
- "robot_ang_vel",
32
- "arm_left_qpos",
33
- "arm_left_qpos_sin",
34
- "arm_left_qpos_cos",
35
- "eef_left_pos",
36
- "eef_left_quat",
37
- "gripper_left_qpos",
38
- "arm_right_qpos",
39
- "arm_right_qpos_sin",
40
- "arm_right_qpos_cos",
41
- "eef_right_pos",
42
- "eef_right_quat",
43
- "gripper_right_qpos",
44
- "trunk_qpos"
45
- ],
46
- "sin_cos_embedding_keys": null,
47
- "mean_std_embedding_keys": null,
48
- "action_configs": null
49
- },
50
- "action": {
51
- "delta_indices": [
52
- 0,
53
- 1,
54
- 2,
55
- 3,
56
- 4,
57
- 5,
58
- 6,
59
- 7,
60
- 8,
61
- 9,
62
- 10,
63
- 11,
64
- 12,
65
- 13,
66
- 14,
67
- 15,
68
- 16,
69
- 17,
70
- 18,
71
- 19,
72
- 20,
73
- 21,
74
- 22,
75
- 23,
76
- 24,
77
- 25,
78
- 26,
79
- 27,
80
- 28,
81
- 29,
82
- 30,
83
- 31
84
- ],
85
- "modality_keys": [
86
- "base",
87
- "torso",
88
- "left_arm",
89
- "left_gripper",
90
- "right_arm",
91
- "right_gripper"
92
- ],
93
- "sin_cos_embedding_keys": null,
94
- "mean_std_embedding_keys": null,
95
- "action_configs": [
96
- {
97
- "rep": "ABSOLUTE",
98
- "type": "NON_EEF",
99
- "format": "DEFAULT",
100
- "state_key": null
101
- },
102
- {
103
- "rep": "RELATIVE",
104
- "type": "NON_EEF",
105
- "format": "DEFAULT",
106
- "state_key": "trunk_qpos"
107
- },
108
- {
109
- "rep": "RELATIVE",
110
- "type": "NON_EEF",
111
- "format": "DEFAULT",
112
- "state_key": "arm_left_qpos"
113
- },
114
- {
115
- "rep": "ABSOLUTE",
116
- "type": "NON_EEF",
117
- "format": "DEFAULT",
118
- "state_key": null
119
- },
120
- {
121
- "rep": "RELATIVE",
122
- "type": "NON_EEF",
123
- "format": "DEFAULT",
124
- "state_key": "arm_right_qpos"
125
- },
126
- {
127
- "rep": "ABSOLUTE",
128
- "type": "NON_EEF",
129
- "format": "DEFAULT",
130
- "state_key": null
131
- }
132
- ]
133
- },
134
- "language": {
135
- "delta_indices": [
136
- 0
137
- ],
138
- "modality_keys": [
139
- "annotation.human.coarse_action"
140
- ],
141
- "sin_cos_embedding_keys": null,
142
- "mean_std_embedding_keys": null,
143
- "action_configs": null
144
- }
145
- },
146
- "gr1": {
147
- "video": {
148
- "delta_indices": [
149
- 0
150
- ],
151
- "modality_keys": [
152
- "ego_view_bg_crop_pad_res256_freq20"
153
- ],
154
- "sin_cos_embedding_keys": null,
155
- "mean_std_embedding_keys": null,
156
- "action_configs": null
157
- },
158
- "state": {
159
- "delta_indices": [
160
- 0
161
- ],
162
- "modality_keys": [
163
- "left_arm",
164
- "right_arm",
165
- "left_hand",
166
- "right_hand",
167
- "waist"
168
- ],
169
- "sin_cos_embedding_keys": [
170
- "left_arm",
171
- "right_arm",
172
- "left_hand",
173
- "right_hand",
174
- "waist"
175
- ],
176
- "mean_std_embedding_keys": null,
177
- "action_configs": null
178
- },
179
- "action": {
180
- "delta_indices": [
181
- 0,
182
- 1,
183
- 2,
184
- 3,
185
- 4,
186
- 5,
187
- 6,
188
- 7,
189
- 8,
190
- 9,
191
- 10,
192
- 11,
193
- 12,
194
- 13,
195
- 14,
196
- 15
197
- ],
198
- "modality_keys": [
199
- "left_arm",
200
- "right_arm",
201
- "left_hand",
202
- "right_hand",
203
- "waist"
204
- ],
205
- "sin_cos_embedding_keys": null,
206
- "mean_std_embedding_keys": null,
207
- "action_configs": [
208
- {
209
- "rep": "RELATIVE",
210
- "type": "NON_EEF",
211
- "format": "DEFAULT",
212
- "state_key": null
213
- },
214
- {
215
- "rep": "RELATIVE",
216
- "type": "NON_EEF",
217
- "format": "DEFAULT",
218
- "state_key": null
219
- },
220
- {
221
- "rep": "RELATIVE",
222
- "type": "NON_EEF",
223
- "format": "DEFAULT",
224
- "state_key": null
225
- },
226
- {
227
- "rep": "RELATIVE",
228
- "type": "NON_EEF",
229
- "format": "DEFAULT",
230
- "state_key": null
231
- },
232
- {
233
- "rep": "ABSOLUTE",
234
- "type": "NON_EEF",
235
- "format": "DEFAULT",
236
- "state_key": null
237
- }
238
- ]
239
- },
240
- "language": {
241
- "delta_indices": [
242
- 0
243
- ],
244
- "modality_keys": [
245
- "task"
246
- ],
247
- "sin_cos_embedding_keys": null,
248
- "mean_std_embedding_keys": null,
249
- "action_configs": null
250
- }
251
- },
252
- "robocasa_panda_omron": {
253
- "video": {
254
- "delta_indices": [
255
- 0
256
- ],
257
- "modality_keys": [
258
- "res256_image_side_0",
259
- "res256_image_side_1",
260
- "res256_image_wrist_0"
261
- ],
262
- "sin_cos_embedding_keys": null,
263
- "mean_std_embedding_keys": null,
264
- "action_configs": null
265
- },
266
- "state": {
267
- "delta_indices": [
268
- 0
269
- ],
270
- "modality_keys": [
271
- "end_effector_position_relative",
272
- "end_effector_rotation_relative",
273
- "gripper_qpos",
274
- "base_position",
275
- "base_rotation"
276
- ],
277
- "sin_cos_embedding_keys": null,
278
- "mean_std_embedding_keys": null,
279
- "action_configs": null
280
- },
281
- "action": {
282
- "delta_indices": [
283
- 0,
284
- 1,
285
- 2,
286
- 3,
287
- 4,
288
- 5,
289
- 6,
290
- 7,
291
- 8,
292
- 9,
293
- 10,
294
- 11,
295
- 12,
296
- 13,
297
- 14,
298
- 15
299
- ],
300
- "modality_keys": [
301
- "end_effector_position",
302
- "end_effector_rotation",
303
- "gripper_close",
304
- "base_motion",
305
- "control_mode"
306
- ],
307
- "sin_cos_embedding_keys": null,
308
- "mean_std_embedding_keys": null,
309
- "action_configs": [
310
- {
311
- "rep": "ABSOLUTE",
312
- "type": "NON_EEF",
313
- "format": "DEFAULT",
314
- "state_key": null
315
- },
316
- {
317
- "rep": "ABSOLUTE",
318
- "type": "NON_EEF",
319
- "format": "DEFAULT",
320
- "state_key": null
321
- },
322
- {
323
- "rep": "ABSOLUTE",
324
- "type": "NON_EEF",
325
- "format": "DEFAULT",
326
- "state_key": null
327
- },
328
- {
329
- "rep": "ABSOLUTE",
330
- "type": "NON_EEF",
331
- "format": "DEFAULT",
332
- "state_key": null
333
- },
334
- {
335
- "rep": "ABSOLUTE",
336
- "type": "NON_EEF",
337
- "format": "DEFAULT",
338
- "state_key": null
339
- }
340
- ]
341
- },
342
- "language": {
343
- "delta_indices": [
344
- 0
345
- ],
346
- "modality_keys": [
347
- "annotation.human.action.task_description"
348
- ],
349
- "sin_cos_embedding_keys": null,
350
- "mean_std_embedding_keys": null,
351
- "action_configs": null
352
- }
353
- },
354
- "unitree_g1": {
355
- "video": {
356
- "delta_indices": [
357
- 0
358
- ],
359
- "modality_keys": [
360
- "ego_view"
361
- ],
362
- "sin_cos_embedding_keys": null,
363
- "mean_std_embedding_keys": null,
364
- "action_configs": null
365
- },
366
- "state": {
367
- "delta_indices": [
368
- 0
369
- ],
370
- "modality_keys": [
371
- "left_leg",
372
- "right_leg",
373
- "waist",
374
- "left_arm",
375
- "right_arm",
376
- "left_hand",
377
- "right_hand"
378
- ],
379
- "sin_cos_embedding_keys": null,
380
- "mean_std_embedding_keys": null,
381
- "action_configs": null
382
- },
383
- "action": {
384
- "delta_indices": [
385
- 0,
386
- 1,
387
- 2,
388
- 3,
389
- 4,
390
- 5,
391
- 6,
392
- 7,
393
- 8,
394
- 9,
395
- 10,
396
- 11,
397
- 12,
398
- 13,
399
- 14,
400
- 15,
401
- 16,
402
- 17,
403
- 18,
404
- 19,
405
- 20,
406
- 21,
407
- 22,
408
- 23,
409
- 24,
410
- 25,
411
- 26,
412
- 27,
413
- 28,
414
- 29
415
- ],
416
- "modality_keys": [
417
- "left_arm",
418
- "right_arm",
419
- "left_hand",
420
- "right_hand",
421
- "waist",
422
- "base_height_command",
423
- "navigate_command"
424
- ],
425
- "sin_cos_embedding_keys": null,
426
- "mean_std_embedding_keys": null,
427
- "action_configs": [
428
- {
429
- "rep": "RELATIVE",
430
- "type": "NON_EEF",
431
- "format": "DEFAULT",
432
- "state_key": null
433
- },
434
- {
435
- "rep": "RELATIVE",
436
- "type": "NON_EEF",
437
- "format": "DEFAULT",
438
- "state_key": null
439
- },
440
- {
441
- "rep": "ABSOLUTE",
442
- "type": "NON_EEF",
443
- "format": "DEFAULT",
444
- "state_key": null
445
- },
446
- {
447
- "rep": "ABSOLUTE",
448
- "type": "NON_EEF",
449
- "format": "DEFAULT",
450
- "state_key": null
451
- },
452
- {
453
- "rep": "ABSOLUTE",
454
- "type": "NON_EEF",
455
- "format": "DEFAULT",
456
- "state_key": null
457
- },
458
- {
459
- "rep": "ABSOLUTE",
460
- "type": "NON_EEF",
461
- "format": "DEFAULT",
462
- "state_key": null
463
- },
464
- {
465
- "rep": "ABSOLUTE",
466
- "type": "NON_EEF",
467
- "format": "DEFAULT",
468
- "state_key": null
469
- }
470
- ]
471
- },
472
- "language": {
473
- "delta_indices": [
474
- 0
475
- ],
476
- "modality_keys": [
477
- "annotation.human.task_description"
478
- ],
479
- "sin_cos_embedding_keys": null,
480
- "mean_std_embedding_keys": null,
481
- "action_configs": null
482
- }
483
- }
484
- },
485
- "image_crop_size": null,
486
- "image_target_size": null,
487
- "use_albumentations": true,
488
- "random_rotation_angle": null,
489
- "color_jitter_params": {
490
- "brightness": 0.3,
491
- "contrast": 0.4,
492
- "saturation": 0.5,
493
- "hue": 0.08
494
- },
495
- "shortest_image_edge": 256,
496
- "crop_fraction": 0.95,
497
- "model_name": "nvidia/Eagle-Block2A-2B-v2",
498
- "model_type": "eagle",
499
- "formalize_language": true,
500
- "max_state_dim": 128,
501
- "max_action_dim": 128,
502
- "max_action_horizon": 50,
503
- "use_percentiles": false,
504
- "clip_outliers": true,
505
- "apply_sincos_state_encoding": true,
506
- "use_relative_action": true
507
- }
508
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-4000/rng_state_0.pth DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:ec6e49911abd2fc83f053e1132fac39188a0cb116e9bcc50d5838ef806a074aa
3
- size 15429
 
 
 
 
checkpoint-4000/rng_state_1.pth DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:e9cb4df49d6d89c24b739036627a9d60424a477c08a177a460a5101db99cac40
3
- size 15429
 
 
 
 
checkpoint-4000/rng_state_2.pth DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:6eedcf354541f2bd34492c7e662db5818f5d506c0fe3a8ba2b4801c0787c351d
3
- size 15429
 
 
 
 
checkpoint-4000/rng_state_3.pth DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:76ff614b9d971d68080f9f1a9a41d0e375160e7d554e294957d190ce405f6763
3
- size 15429
 
 
 
 
checkpoint-4000/scheduler.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:93727dfaa883ba52dca9378688334804bd7142a7af39e64fe8ca4bf76f6b302c
3
- size 1465
 
 
 
 
checkpoint-4000/statistics.json DELETED
The diff for this file is too large to render. See raw diff
 
checkpoint-4000/trainer_state.json DELETED
@@ -1,2434 +0,0 @@
1
- {
2
- "best_global_step": null,
3
- "best_metric": null,
4
- "best_model_checkpoint": null,
5
- "epoch": 0.6666666666666666,
6
- "eval_steps": 500,
7
- "global_step": 4000,
8
- "is_hyper_param_search": false,
9
- "is_local_process_zero": true,
10
- "is_world_process_zero": true,
11
- "log_history": [
12
- {
13
- "grad_norm": 1.3613678216934204,
14
- "learning_rate": 3e-06,
15
- "loss": 1.4557,
16
- "step": 10
17
- },
18
- {
19
- "grad_norm": 0.8573107719421387,
20
- "learning_rate": 6.333333333333334e-06,
21
- "loss": 1.424,
22
- "step": 20
23
- },
24
- {
25
- "grad_norm": 0.34049543738365173,
26
- "learning_rate": 9.666666666666667e-06,
27
- "loss": 1.3391,
28
- "step": 30
29
- },
30
- {
31
- "grad_norm": 0.44315752387046814,
32
- "learning_rate": 1.3000000000000001e-05,
33
- "loss": 1.2758,
34
- "step": 40
35
- },
36
- {
37
- "grad_norm": 0.49191826581954956,
38
- "learning_rate": 1.6333333333333335e-05,
39
- "loss": 1.175,
40
- "step": 50
41
- },
42
- {
43
- "grad_norm": 0.3820122480392456,
44
- "learning_rate": 1.9666666666666666e-05,
45
- "loss": 1.1102,
46
- "step": 60
47
- },
48
- {
49
- "grad_norm": 0.36782893538475037,
50
- "learning_rate": 2.3000000000000003e-05,
51
- "loss": 1.0885,
52
- "step": 70
53
- },
54
- {
55
- "grad_norm": 0.28842034935951233,
56
- "learning_rate": 2.633333333333333e-05,
57
- "loss": 1.0746,
58
- "step": 80
59
- },
60
- {
61
- "grad_norm": 0.43426793813705444,
62
- "learning_rate": 2.9666666666666672e-05,
63
- "loss": 1.0604,
64
- "step": 90
65
- },
66
- {
67
- "grad_norm": 0.34185388684272766,
68
- "learning_rate": 3.3e-05,
69
- "loss": 1.0523,
70
- "step": 100
71
- },
72
- {
73
- "grad_norm": 0.29589465260505676,
74
- "learning_rate": 3.633333333333333e-05,
75
- "loss": 1.0355,
76
- "step": 110
77
- },
78
- {
79
- "grad_norm": 0.42089858651161194,
80
- "learning_rate": 3.966666666666667e-05,
81
- "loss": 1.0121,
82
- "step": 120
83
- },
84
- {
85
- "grad_norm": 0.4485030472278595,
86
- "learning_rate": 4.3e-05,
87
- "loss": 0.98,
88
- "step": 130
89
- },
90
- {
91
- "grad_norm": 0.3440701365470886,
92
- "learning_rate": 4.633333333333333e-05,
93
- "loss": 0.9491,
94
- "step": 140
95
- },
96
- {
97
- "grad_norm": 0.7382627725601196,
98
- "learning_rate": 4.966666666666667e-05,
99
- "loss": 0.9054,
100
- "step": 150
101
- },
102
- {
103
- "grad_norm": 0.6498928070068359,
104
- "learning_rate": 5.300000000000001e-05,
105
- "loss": 0.8479,
106
- "step": 160
107
- },
108
- {
109
- "grad_norm": 0.6362508535385132,
110
- "learning_rate": 5.633333333333334e-05,
111
- "loss": 0.7839,
112
- "step": 170
113
- },
114
- {
115
- "grad_norm": 0.8518823385238647,
116
- "learning_rate": 5.966666666666667e-05,
117
- "loss": 0.7336,
118
- "step": 180
119
- },
120
- {
121
- "grad_norm": 1.1828296184539795,
122
- "learning_rate": 6.3e-05,
123
- "loss": 0.6629,
124
- "step": 190
125
- },
126
- {
127
- "grad_norm": 0.944200336933136,
128
- "learning_rate": 6.633333333333334e-05,
129
- "loss": 0.6056,
130
- "step": 200
131
- },
132
- {
133
- "grad_norm": 0.758030354976654,
134
- "learning_rate": 6.966666666666668e-05,
135
- "loss": 0.5545,
136
- "step": 210
137
- },
138
- {
139
- "grad_norm": 1.2042676210403442,
140
- "learning_rate": 7.3e-05,
141
- "loss": 0.5044,
142
- "step": 220
143
- },
144
- {
145
- "grad_norm": 1.0285093784332275,
146
- "learning_rate": 7.633333333333334e-05,
147
- "loss": 0.4607,
148
- "step": 230
149
- },
150
- {
151
- "grad_norm": 1.126552700996399,
152
- "learning_rate": 7.966666666666666e-05,
153
- "loss": 0.4219,
154
- "step": 240
155
- },
156
- {
157
- "grad_norm": 0.7760715484619141,
158
- "learning_rate": 8.3e-05,
159
- "loss": 0.3931,
160
- "step": 250
161
- },
162
- {
163
- "grad_norm": 1.1023366451263428,
164
- "learning_rate": 8.633333333333334e-05,
165
- "loss": 0.362,
166
- "step": 260
167
- },
168
- {
169
- "grad_norm": 0.9510546326637268,
170
- "learning_rate": 8.966666666666666e-05,
171
- "loss": 0.3313,
172
- "step": 270
173
- },
174
- {
175
- "grad_norm": 0.8212385177612305,
176
- "learning_rate": 9.300000000000001e-05,
177
- "loss": 0.2998,
178
- "step": 280
179
- },
180
- {
181
- "grad_norm": 0.8118603229522705,
182
- "learning_rate": 9.633333333333335e-05,
183
- "loss": 0.28,
184
- "step": 290
185
- },
186
- {
187
- "grad_norm": 1.409279704093933,
188
- "learning_rate": 9.966666666666667e-05,
189
- "loss": 0.2516,
190
- "step": 300
191
- },
192
- {
193
- "grad_norm": 2.358229398727417,
194
- "learning_rate": 9.999938485971279e-05,
195
- "loss": 0.2319,
196
- "step": 310
197
- },
198
- {
199
- "grad_norm": 0.9309172630310059,
200
- "learning_rate": 9.999725846827562e-05,
201
- "loss": 0.2257,
202
- "step": 320
203
- },
204
- {
205
- "grad_norm": 0.9242991805076599,
206
- "learning_rate": 9.999361329594254e-05,
207
- "loss": 0.209,
208
- "step": 330
209
- },
210
- {
211
- "grad_norm": 0.9545067548751831,
212
- "learning_rate": 9.998844945344405e-05,
213
- "loss": 0.1922,
214
- "step": 340
215
- },
216
- {
217
- "grad_norm": 0.9418713450431824,
218
- "learning_rate": 9.99817670976436e-05,
219
- "loss": 0.175,
220
- "step": 350
221
- },
222
- {
223
- "grad_norm": 0.8053094744682312,
224
- "learning_rate": 9.997356643153303e-05,
225
- "loss": 0.161,
226
- "step": 360
227
- },
228
- {
229
- "grad_norm": 1.237301230430603,
230
- "learning_rate": 9.996384770422629e-05,
231
- "loss": 0.1531,
232
- "step": 370
233
- },
234
- {
235
- "grad_norm": 1.0992234945297241,
236
- "learning_rate": 9.995261121095194e-05,
237
- "loss": 0.1391,
238
- "step": 380
239
- },
240
- {
241
- "grad_norm": 1.061476707458496,
242
- "learning_rate": 9.993985729304408e-05,
243
- "loss": 0.1302,
244
- "step": 390
245
- },
246
- {
247
- "grad_norm": 0.8854357004165649,
248
- "learning_rate": 9.992558633793212e-05,
249
- "loss": 0.1156,
250
- "step": 400
251
- },
252
- {
253
- "grad_norm": 0.7982974648475647,
254
- "learning_rate": 9.990979877912891e-05,
255
- "loss": 0.108,
256
- "step": 410
257
- },
258
- {
259
- "grad_norm": 0.8326540589332581,
260
- "learning_rate": 9.989249509621759e-05,
261
- "loss": 0.1122,
262
- "step": 420
263
- },
264
- {
265
- "grad_norm": 0.7788134813308716,
266
- "learning_rate": 9.987367581483705e-05,
267
- "loss": 0.1114,
268
- "step": 430
269
- },
270
- {
271
- "grad_norm": 0.8146352767944336,
272
- "learning_rate": 9.985334150666592e-05,
273
- "loss": 0.1074,
274
- "step": 440
275
- },
276
- {
277
- "grad_norm": 0.745625913143158,
278
- "learning_rate": 9.983149278940526e-05,
279
- "loss": 0.105,
280
- "step": 450
281
- },
282
- {
283
- "grad_norm": 0.9119800329208374,
284
- "learning_rate": 9.980813032675974e-05,
285
- "loss": 0.099,
286
- "step": 460
287
- },
288
- {
289
- "grad_norm": 0.6874001622200012,
290
- "learning_rate": 9.978325482841753e-05,
291
- "loss": 0.0983,
292
- "step": 470
293
- },
294
- {
295
- "grad_norm": 1.0723556280136108,
296
- "learning_rate": 9.975686705002867e-05,
297
- "loss": 0.0953,
298
- "step": 480
299
- },
300
- {
301
- "grad_norm": 0.7956628203392029,
302
- "learning_rate": 9.972896779318219e-05,
303
- "loss": 0.0813,
304
- "step": 490
305
- },
306
- {
307
- "grad_norm": 0.8455336093902588,
308
- "learning_rate": 9.969955790538175e-05,
309
- "loss": 0.0781,
310
- "step": 500
311
- },
312
- {
313
- "grad_norm": 0.8610921502113342,
314
- "learning_rate": 9.966863828001982e-05,
315
- "loss": 0.0751,
316
- "step": 510
317
- },
318
- {
319
- "grad_norm": 0.7537756562232971,
320
- "learning_rate": 9.963620985635065e-05,
321
- "loss": 0.0692,
322
- "step": 520
323
- },
324
- {
325
- "grad_norm": 0.7432633638381958,
326
- "learning_rate": 9.960227361946164e-05,
327
- "loss": 0.0664,
328
- "step": 530
329
- },
330
- {
331
- "grad_norm": 0.9131788611412048,
332
- "learning_rate": 9.95668306002435e-05,
333
- "loss": 0.0691,
334
- "step": 540
335
- },
336
- {
337
- "grad_norm": 0.913440465927124,
338
- "learning_rate": 9.952988187535886e-05,
339
- "loss": 0.0665,
340
- "step": 550
341
- },
342
- {
343
- "grad_norm": 0.7507103681564331,
344
- "learning_rate": 9.949142856720961e-05,
345
- "loss": 0.0641,
346
- "step": 560
347
- },
348
- {
349
- "grad_norm": 0.7820024490356445,
350
- "learning_rate": 9.945147184390278e-05,
351
- "loss": 0.0695,
352
- "step": 570
353
- },
354
- {
355
- "grad_norm": 0.7565670609474182,
356
- "learning_rate": 9.941001291921512e-05,
357
- "loss": 0.0663,
358
- "step": 580
359
- },
360
- {
361
- "grad_norm": 0.6481784582138062,
362
- "learning_rate": 9.936705305255612e-05,
363
- "loss": 0.0661,
364
- "step": 590
365
- },
366
- {
367
- "grad_norm": 0.6356178522109985,
368
- "learning_rate": 9.932259354892984e-05,
369
- "loss": 0.0641,
370
- "step": 600
371
- },
372
- {
373
- "grad_norm": 0.7873478531837463,
374
- "learning_rate": 9.927663575889521e-05,
375
- "loss": 0.062,
376
- "step": 610
377
- },
378
- {
379
- "grad_norm": 0.7071146368980408,
380
- "learning_rate": 9.922918107852504e-05,
381
- "loss": 0.0639,
382
- "step": 620
383
- },
384
- {
385
- "grad_norm": 0.8344459533691406,
386
- "learning_rate": 9.918023094936363e-05,
387
- "loss": 0.0607,
388
- "step": 630
389
- },
390
- {
391
- "grad_norm": 0.7432113885879517,
392
- "learning_rate": 9.912978685838294e-05,
393
- "loss": 0.0622,
394
- "step": 640
395
- },
396
- {
397
- "grad_norm": 0.7232807278633118,
398
- "learning_rate": 9.90778503379374e-05,
399
- "loss": 0.0595,
400
- "step": 650
401
- },
402
- {
403
- "grad_norm": 0.705303966999054,
404
- "learning_rate": 9.902442296571743e-05,
405
- "loss": 0.0571,
406
- "step": 660
407
- },
408
- {
409
- "grad_norm": 0.7458016872406006,
410
- "learning_rate": 9.896950636470147e-05,
411
- "loss": 0.0652,
412
- "step": 670
413
- },
414
- {
415
- "grad_norm": 0.7948721647262573,
416
- "learning_rate": 9.891310220310666e-05,
417
- "loss": 0.0596,
418
- "step": 680
419
- },
420
- {
421
- "grad_norm": 0.78054279088974,
422
- "learning_rate": 9.885521219433823e-05,
423
- "loss": 0.0564,
424
- "step": 690
425
- },
426
- {
427
- "grad_norm": 0.7607908248901367,
428
- "learning_rate": 9.879583809693738e-05,
429
- "loss": 0.0565,
430
- "step": 700
431
- },
432
- {
433
- "grad_norm": 0.6699449419975281,
434
- "learning_rate": 9.873498171452789e-05,
435
- "loss": 0.0553,
436
- "step": 710
437
- },
438
- {
439
- "grad_norm": 0.7117671370506287,
440
- "learning_rate": 9.867264489576135e-05,
441
- "loss": 0.0555,
442
- "step": 720
443
- },
444
- {
445
- "grad_norm": 0.8966276049613953,
446
- "learning_rate": 9.860882953426099e-05,
447
- "loss": 0.0576,
448
- "step": 730
449
- },
450
- {
451
- "grad_norm": 0.8119339346885681,
452
- "learning_rate": 9.854353756856412e-05,
453
- "loss": 0.0571,
454
- "step": 740
455
- },
456
- {
457
- "grad_norm": 0.8719547390937805,
458
- "learning_rate": 9.847677098206332e-05,
459
- "loss": 0.0554,
460
- "step": 750
461
- },
462
- {
463
- "grad_norm": 0.6648759245872498,
464
- "learning_rate": 9.840853180294608e-05,
465
- "loss": 0.0551,
466
- "step": 760
467
- },
468
- {
469
- "grad_norm": 0.8022528290748596,
470
- "learning_rate": 9.833882210413332e-05,
471
- "loss": 0.0562,
472
- "step": 770
473
- },
474
- {
475
- "grad_norm": 0.6207597255706787,
476
- "learning_rate": 9.826764400321633e-05,
477
- "loss": 0.0553,
478
- "step": 780
479
- },
480
- {
481
- "grad_norm": 0.7227312922477722,
482
- "learning_rate": 9.819499966239243e-05,
483
- "loss": 0.0544,
484
- "step": 790
485
- },
486
- {
487
- "grad_norm": 0.5820620059967041,
488
- "learning_rate": 9.812089128839938e-05,
489
- "loss": 0.0544,
490
- "step": 800
491
- },
492
- {
493
- "grad_norm": 0.6752761602401733,
494
- "learning_rate": 9.804532113244828e-05,
495
- "loss": 0.0511,
496
- "step": 810
497
- },
498
- {
499
- "grad_norm": 0.650572657585144,
500
- "learning_rate": 9.796829149015517e-05,
501
- "loss": 0.0529,
502
- "step": 820
503
- },
504
- {
505
- "grad_norm": 0.6513978838920593,
506
- "learning_rate": 9.788980470147132e-05,
507
- "loss": 0.05,
508
- "step": 830
509
- },
510
- {
511
- "grad_norm": 0.6479138731956482,
512
- "learning_rate": 9.780986315061218e-05,
513
- "loss": 0.0518,
514
- "step": 840
515
- },
516
- {
517
- "grad_norm": 0.7067680358886719,
518
- "learning_rate": 9.772846926598491e-05,
519
- "loss": 0.0568,
520
- "step": 850
521
- },
522
- {
523
- "grad_norm": 0.6965780258178711,
524
- "learning_rate": 9.76456255201146e-05,
525
- "loss": 0.052,
526
- "step": 860
527
- },
528
- {
529
- "grad_norm": 0.6729057431221008,
530
- "learning_rate": 9.756133442956923e-05,
531
- "loss": 0.0502,
532
- "step": 870
533
- },
534
- {
535
- "grad_norm": 0.5427769422531128,
536
- "learning_rate": 9.747559855488313e-05,
537
- "loss": 0.0513,
538
- "step": 880
539
- },
540
- {
541
- "grad_norm": 0.6974294781684875,
542
- "learning_rate": 9.73884205004793e-05,
543
- "loss": 0.0526,
544
- "step": 890
545
- },
546
- {
547
- "grad_norm": 0.6217942833900452,
548
- "learning_rate": 9.729980291459019e-05,
549
- "loss": 0.0523,
550
- "step": 900
551
- },
552
- {
553
- "grad_norm": 0.7760837078094482,
554
- "learning_rate": 9.720974848917735e-05,
555
- "loss": 0.0498,
556
- "step": 910
557
- },
558
- {
559
- "grad_norm": 0.624162495136261,
560
- "learning_rate": 9.711825995984957e-05,
561
- "loss": 0.0499,
562
- "step": 920
563
- },
564
- {
565
- "grad_norm": 0.6548763513565063,
566
- "learning_rate": 9.702534010577991e-05,
567
- "loss": 0.0507,
568
- "step": 930
569
- },
570
- {
571
- "grad_norm": 0.74265056848526,
572
- "learning_rate": 9.693099174962103e-05,
573
- "loss": 0.0512,
574
- "step": 940
575
- },
576
- {
577
- "grad_norm": 0.5611060857772827,
578
- "learning_rate": 9.683521775741977e-05,
579
- "loss": 0.0514,
580
- "step": 950
581
- },
582
- {
583
- "grad_norm": 0.7143239974975586,
584
- "learning_rate": 9.673802103852979e-05,
585
- "loss": 0.0493,
586
- "step": 960
587
- },
588
- {
589
- "grad_norm": 0.6779119968414307,
590
- "learning_rate": 9.663940454552342e-05,
591
- "loss": 0.0499,
592
- "step": 970
593
- },
594
- {
595
- "grad_norm": 0.6951384544372559,
596
- "learning_rate": 9.65393712741018e-05,
597
- "loss": 0.0474,
598
- "step": 980
599
- },
600
- {
601
- "grad_norm": 0.6736275553703308,
602
- "learning_rate": 9.6437924263004e-05,
603
- "loss": 0.0485,
604
- "step": 990
605
- },
606
- {
607
- "grad_norm": 0.5961176156997681,
608
- "learning_rate": 9.63350665939146e-05,
609
- "loss": 0.0477,
610
- "step": 1000
611
- },
612
- {
613
- "grad_norm": 0.7116590142250061,
614
- "learning_rate": 9.623080139137023e-05,
615
- "loss": 0.0466,
616
- "step": 1010
617
- },
618
- {
619
- "grad_norm": 0.5824304223060608,
620
- "learning_rate": 9.612513182266447e-05,
621
- "loss": 0.0475,
622
- "step": 1020
623
- },
624
- {
625
- "grad_norm": 0.8167940378189087,
626
- "learning_rate": 9.601806109775179e-05,
627
- "loss": 0.0485,
628
- "step": 1030
629
- },
630
- {
631
- "grad_norm": 0.6272838711738586,
632
- "learning_rate": 9.590959246914995e-05,
633
- "loss": 0.0466,
634
- "step": 1040
635
- },
636
- {
637
- "grad_norm": 0.8012779951095581,
638
- "learning_rate": 9.579972923184122e-05,
639
- "loss": 0.0487,
640
- "step": 1050
641
- },
642
- {
643
- "grad_norm": 0.5613420009613037,
644
- "learning_rate": 9.568847472317232e-05,
645
- "loss": 0.0448,
646
- "step": 1060
647
- },
648
- {
649
- "grad_norm": 0.7028784155845642,
650
- "learning_rate": 9.557583232275303e-05,
651
- "loss": 0.0474,
652
- "step": 1070
653
- },
654
- {
655
- "grad_norm": 0.705496609210968,
656
- "learning_rate": 9.546180545235344e-05,
657
- "loss": 0.0456,
658
- "step": 1080
659
- },
660
- {
661
- "grad_norm": 0.6239430904388428,
662
- "learning_rate": 9.534639757580013e-05,
663
- "loss": 0.0442,
664
- "step": 1090
665
- },
666
- {
667
- "grad_norm": 0.6760733723640442,
668
- "learning_rate": 9.522961219887092e-05,
669
- "loss": 0.046,
670
- "step": 1100
671
- },
672
- {
673
- "grad_norm": 0.6449405550956726,
674
- "learning_rate": 9.511145286918828e-05,
675
- "loss": 0.0475,
676
- "step": 1110
677
- },
678
- {
679
- "grad_norm": 0.5605469346046448,
680
- "learning_rate": 9.499192317611167e-05,
681
- "loss": 0.0438,
682
- "step": 1120
683
- },
684
- {
685
- "grad_norm": 0.6307807564735413,
686
- "learning_rate": 9.487102675062851e-05,
687
- "loss": 0.0468,
688
- "step": 1130
689
- },
690
- {
691
- "grad_norm": 0.5671112537384033,
692
- "learning_rate": 9.474876726524374e-05,
693
- "loss": 0.0426,
694
- "step": 1140
695
- },
696
- {
697
- "grad_norm": 0.8449708223342896,
698
- "learning_rate": 9.462514843386845e-05,
699
- "loss": 0.0434,
700
- "step": 1150
701
- },
702
- {
703
- "grad_norm": 0.6328504085540771,
704
- "learning_rate": 9.450017401170689e-05,
705
- "loss": 0.0464,
706
- "step": 1160
707
- },
708
- {
709
- "grad_norm": 0.5694561004638672,
710
- "learning_rate": 9.437384779514256e-05,
711
- "loss": 0.0462,
712
- "step": 1170
713
- },
714
- {
715
- "grad_norm": 0.5980640053749084,
716
- "learning_rate": 9.424617362162271e-05,
717
- "loss": 0.0465,
718
- "step": 1180
719
- },
720
- {
721
- "grad_norm": 0.6236596703529358,
722
- "learning_rate": 9.411715536954196e-05,
723
- "loss": 0.0434,
724
- "step": 1190
725
- },
726
- {
727
- "grad_norm": 0.5503287315368652,
728
- "learning_rate": 9.39867969581243e-05,
729
- "loss": 0.0438,
730
- "step": 1200
731
- },
732
- {
733
- "grad_norm": 0.7289694547653198,
734
- "learning_rate": 9.385510234730415e-05,
735
- "loss": 0.0452,
736
- "step": 1210
737
- },
738
- {
739
- "grad_norm": 0.5549660325050354,
740
- "learning_rate": 9.372207553760603e-05,
741
- "loss": 0.0435,
742
- "step": 1220
743
- },
744
- {
745
- "grad_norm": 0.6124976277351379,
746
- "learning_rate": 9.358772057002312e-05,
747
- "loss": 0.0423,
748
- "step": 1230
749
- },
750
- {
751
- "grad_norm": 0.6705859899520874,
752
- "learning_rate": 9.345204152589428e-05,
753
- "loss": 0.0425,
754
- "step": 1240
755
- },
756
- {
757
- "grad_norm": 0.5378178954124451,
758
- "learning_rate": 9.331504252678037e-05,
759
- "loss": 0.045,
760
- "step": 1250
761
- },
762
- {
763
- "grad_norm": 0.729077935218811,
764
- "learning_rate": 9.317672773433876e-05,
765
- "loss": 0.0431,
766
- "step": 1260
767
- },
768
- {
769
- "grad_norm": 0.5443227887153625,
770
- "learning_rate": 9.30371013501972e-05,
771
- "loss": 0.0436,
772
- "step": 1270
773
- },
774
- {
775
- "grad_norm": 0.6833415031433105,
776
- "learning_rate": 9.289616761582587e-05,
777
- "loss": 0.0461,
778
- "step": 1280
779
- },
780
- {
781
- "grad_norm": 0.6458922028541565,
782
- "learning_rate": 9.275393081240882e-05,
783
- "loss": 0.0424,
784
- "step": 1290
785
- },
786
- {
787
- "grad_norm": 0.726543664932251,
788
- "learning_rate": 9.261039526071374e-05,
789
- "loss": 0.0447,
790
- "step": 1300
791
- },
792
- {
793
- "grad_norm": 0.5732786655426025,
794
- "learning_rate": 9.246556532096078e-05,
795
- "loss": 0.0431,
796
- "step": 1310
797
- },
798
- {
799
- "grad_norm": 0.7566348910331726,
800
- "learning_rate": 9.231944539269009e-05,
801
- "loss": 0.0436,
802
- "step": 1320
803
- },
804
- {
805
- "grad_norm": 0.772743284702301,
806
- "learning_rate": 9.217203991462815e-05,
807
- "loss": 0.0435,
808
- "step": 1330
809
- },
810
- {
811
- "grad_norm": 0.6129595637321472,
812
- "learning_rate": 9.202335336455296e-05,
813
- "loss": 0.0423,
814
- "step": 1340
815
- },
816
- {
817
- "grad_norm": 0.5188651084899902,
818
- "learning_rate": 9.187339025915802e-05,
819
- "loss": 0.0411,
820
- "step": 1350
821
- },
822
- {
823
- "grad_norm": 0.6343339085578918,
824
- "learning_rate": 9.17221551539151e-05,
825
- "loss": 0.0397,
826
- "step": 1360
827
- },
828
- {
829
- "grad_norm": 0.6620864868164062,
830
- "learning_rate": 9.156965264293586e-05,
831
- "loss": 0.0409,
832
- "step": 1370
833
- },
834
- {
835
- "grad_norm": 0.6305266618728638,
836
- "learning_rate": 9.141588735883232e-05,
837
- "loss": 0.0393,
838
- "step": 1380
839
- },
840
- {
841
- "grad_norm": 0.7356075644493103,
842
- "learning_rate": 9.126086397257612e-05,
843
- "loss": 0.0417,
844
- "step": 1390
845
- },
846
- {
847
- "grad_norm": 0.5255911350250244,
848
- "learning_rate": 9.110458719335659e-05,
849
- "loss": 0.0409,
850
- "step": 1400
851
- },
852
- {
853
- "grad_norm": 0.582068920135498,
854
- "learning_rate": 9.094706176843777e-05,
855
- "loss": 0.0424,
856
- "step": 1410
857
- },
858
- {
859
- "grad_norm": 0.48159053921699524,
860
- "learning_rate": 9.078829248301417e-05,
861
- "loss": 0.039,
862
- "step": 1420
863
- },
864
- {
865
- "grad_norm": 0.6647977232933044,
866
- "learning_rate": 9.062828416006539e-05,
867
- "loss": 0.0424,
868
- "step": 1430
869
- },
870
- {
871
- "grad_norm": 0.6569739580154419,
872
- "learning_rate": 9.046704166020961e-05,
873
- "loss": 0.0405,
874
- "step": 1440
875
- },
876
- {
877
- "grad_norm": 0.5740228891372681,
878
- "learning_rate": 9.030456988155596e-05,
879
- "loss": 0.0403,
880
- "step": 1450
881
- },
882
- {
883
- "grad_norm": 0.5312358140945435,
884
- "learning_rate": 9.014087375955573e-05,
885
- "loss": 0.04,
886
- "step": 1460
887
- },
888
- {
889
- "grad_norm": 0.6075960993766785,
890
- "learning_rate": 8.997595826685243e-05,
891
- "loss": 0.0447,
892
- "step": 1470
893
- },
894
- {
895
- "grad_norm": 0.4863235056400299,
896
- "learning_rate": 8.980982841313074e-05,
897
- "loss": 0.04,
898
- "step": 1480
899
- },
900
- {
901
- "grad_norm": 0.5700269937515259,
902
- "learning_rate": 8.964248924496435e-05,
903
- "loss": 0.0381,
904
- "step": 1490
905
- },
906
- {
907
- "grad_norm": 0.6507738828659058,
908
- "learning_rate": 8.947394584566258e-05,
909
- "loss": 0.0389,
910
- "step": 1500
911
- },
912
- {
913
- "grad_norm": 0.5595966577529907,
914
- "learning_rate": 8.930420333511606e-05,
915
- "loss": 0.0391,
916
- "step": 1510
917
- },
918
- {
919
- "grad_norm": 0.6669788360595703,
920
- "learning_rate": 8.913326686964117e-05,
921
- "loss": 0.0389,
922
- "step": 1520
923
- },
924
- {
925
- "grad_norm": 0.6669943332672119,
926
- "learning_rate": 8.89611416418234e-05,
927
- "loss": 0.0383,
928
- "step": 1530
929
- },
930
- {
931
- "grad_norm": 0.6302319765090942,
932
- "learning_rate": 8.878783288035957e-05,
933
- "loss": 0.0387,
934
- "step": 1540
935
- },
936
- {
937
- "grad_norm": 0.5663589239120483,
938
- "learning_rate": 8.86133458498991e-05,
939
- "loss": 0.0409,
940
- "step": 1550
941
- },
942
- {
943
- "grad_norm": 0.6851902008056641,
944
- "learning_rate": 8.843768585088393e-05,
945
- "loss": 0.039,
946
- "step": 1560
947
- },
948
- {
949
- "grad_norm": 0.4790905714035034,
950
- "learning_rate": 8.82608582193877e-05,
951
- "loss": 0.0397,
952
- "step": 1570
953
- },
954
- {
955
- "grad_norm": 0.5178065896034241,
956
- "learning_rate": 8.80828683269535e-05,
957
- "loss": 0.0378,
958
- "step": 1580
959
- },
960
- {
961
- "grad_norm": 0.5918937921524048,
962
- "learning_rate": 8.790372158043074e-05,
963
- "loss": 0.0372,
964
- "step": 1590
965
- },
966
- {
967
- "grad_norm": 0.5967026948928833,
968
- "learning_rate": 8.772342342181095e-05,
969
- "loss": 0.0382,
970
- "step": 1600
971
- },
972
- {
973
- "grad_norm": 0.6606245636940002,
974
- "learning_rate": 8.75419793280624e-05,
975
- "loss": 0.0375,
976
- "step": 1610
977
- },
978
- {
979
- "grad_norm": 0.6581755876541138,
980
- "learning_rate": 8.735939481096378e-05,
981
- "loss": 0.0376,
982
- "step": 1620
983
- },
984
- {
985
- "grad_norm": 0.6129235029220581,
986
- "learning_rate": 8.717567541693673e-05,
987
- "loss": 0.0394,
988
- "step": 1630
989
- },
990
- {
991
- "grad_norm": 0.6601637601852417,
992
- "learning_rate": 8.699082672687734e-05,
993
- "loss": 0.0389,
994
- "step": 1640
995
- },
996
- {
997
- "grad_norm": 0.5138643980026245,
998
- "learning_rate": 8.680485435598673e-05,
999
- "loss": 0.0423,
1000
- "step": 1650
1001
- },
1002
- {
1003
- "grad_norm": 0.48632222414016724,
1004
- "learning_rate": 8.661776395360029e-05,
1005
- "loss": 0.0391,
1006
- "step": 1660
1007
- },
1008
- {
1009
- "grad_norm": 0.7064632773399353,
1010
- "learning_rate": 8.642956120301626e-05,
1011
- "loss": 0.0372,
1012
- "step": 1670
1013
- },
1014
- {
1015
- "grad_norm": 0.599163293838501,
1016
- "learning_rate": 8.624025182132292e-05,
1017
- "loss": 0.0387,
1018
- "step": 1680
1019
- },
1020
- {
1021
- "grad_norm": 0.49687978625297546,
1022
- "learning_rate": 8.604984155922506e-05,
1023
- "loss": 0.0384,
1024
- "step": 1690
1025
- },
1026
- {
1027
- "grad_norm": 0.5044621229171753,
1028
- "learning_rate": 8.585833620086918e-05,
1029
- "loss": 0.037,
1030
- "step": 1700
1031
- },
1032
- {
1033
- "grad_norm": 0.48257848620414734,
1034
- "learning_rate": 8.566574156366784e-05,
1035
- "loss": 0.0355,
1036
- "step": 1710
1037
- },
1038
- {
1039
- "grad_norm": 0.7500448822975159,
1040
- "learning_rate": 8.547206349812298e-05,
1041
- "loss": 0.0396,
1042
- "step": 1720
1043
- },
1044
- {
1045
- "grad_norm": 0.5865467190742493,
1046
- "learning_rate": 8.527730788764805e-05,
1047
- "loss": 0.0392,
1048
- "step": 1730
1049
- },
1050
- {
1051
- "grad_norm": 0.5636748671531677,
1052
- "learning_rate": 8.508148064838948e-05,
1053
- "loss": 0.0362,
1054
- "step": 1740
1055
- },
1056
- {
1057
- "grad_norm": 0.6705951690673828,
1058
- "learning_rate": 8.488458772904684e-05,
1059
- "loss": 0.0366,
1060
- "step": 1750
1061
- },
1062
- {
1063
- "grad_norm": 0.5554336309432983,
1064
- "learning_rate": 8.468663511069217e-05,
1065
- "loss": 0.0363,
1066
- "step": 1760
1067
- },
1068
- {
1069
- "grad_norm": 0.45949798822402954,
1070
- "learning_rate": 8.448762880658825e-05,
1071
- "loss": 0.0383,
1072
- "step": 1770
1073
- },
1074
- {
1075
- "grad_norm": 0.5537176132202148,
1076
- "learning_rate": 8.428757486200603e-05,
1077
- "loss": 0.0367,
1078
- "step": 1780
1079
- },
1080
- {
1081
- "grad_norm": 0.4168303608894348,
1082
- "learning_rate": 8.40864793540409e-05,
1083
- "loss": 0.0396,
1084
- "step": 1790
1085
- },
1086
- {
1087
- "grad_norm": 0.6133352518081665,
1088
- "learning_rate": 8.388434839142813e-05,
1089
- "loss": 0.0378,
1090
- "step": 1800
1091
- },
1092
- {
1093
- "grad_norm": 0.484385222196579,
1094
- "learning_rate": 8.368118811435726e-05,
1095
- "loss": 0.035,
1096
- "step": 1810
1097
- },
1098
- {
1099
- "grad_norm": 0.4967355728149414,
1100
- "learning_rate": 8.347700469428564e-05,
1101
- "loss": 0.0351,
1102
- "step": 1820
1103
- },
1104
- {
1105
- "grad_norm": 0.5309926867485046,
1106
- "learning_rate": 8.327180433375091e-05,
1107
- "loss": 0.0364,
1108
- "step": 1830
1109
- },
1110
- {
1111
- "grad_norm": 0.5246663093566895,
1112
- "learning_rate": 8.306559326618259e-05,
1113
- "loss": 0.0358,
1114
- "step": 1840
1115
- },
1116
- {
1117
- "grad_norm": 0.5756788849830627,
1118
- "learning_rate": 8.285837775571276e-05,
1119
- "loss": 0.0357,
1120
- "step": 1850
1121
- },
1122
- {
1123
- "grad_norm": 0.5469930768013,
1124
- "learning_rate": 8.265016409698573e-05,
1125
- "loss": 0.0365,
1126
- "step": 1860
1127
- },
1128
- {
1129
- "grad_norm": 0.5380656123161316,
1130
- "learning_rate": 8.244095861496686e-05,
1131
- "loss": 0.036,
1132
- "step": 1870
1133
- },
1134
- {
1135
- "grad_norm": 0.45056721568107605,
1136
- "learning_rate": 8.223076766475035e-05,
1137
- "loss": 0.0358,
1138
- "step": 1880
1139
- },
1140
- {
1141
- "grad_norm": 0.7063160538673401,
1142
- "learning_rate": 8.201959763136633e-05,
1143
- "loss": 0.0344,
1144
- "step": 1890
1145
- },
1146
- {
1147
- "grad_norm": 0.5377547144889832,
1148
- "learning_rate": 8.180745492958674e-05,
1149
- "loss": 0.0364,
1150
- "step": 1900
1151
- },
1152
- {
1153
- "grad_norm": 0.4956304132938385,
1154
- "learning_rate": 8.159434600373061e-05,
1155
- "loss": 0.0335,
1156
- "step": 1910
1157
- },
1158
- {
1159
- "grad_norm": 0.650954008102417,
1160
- "learning_rate": 8.138027732746818e-05,
1161
- "loss": 0.0357,
1162
- "step": 1920
1163
- },
1164
- {
1165
- "grad_norm": 0.43918663263320923,
1166
- "learning_rate": 8.116525540362434e-05,
1167
- "loss": 0.0341,
1168
- "step": 1930
1169
- },
1170
- {
1171
- "grad_norm": 0.4600161612033844,
1172
- "learning_rate": 8.094928676398101e-05,
1173
- "loss": 0.0348,
1174
- "step": 1940
1175
- },
1176
- {
1177
- "grad_norm": 0.5328519344329834,
1178
- "learning_rate": 8.073237796907882e-05,
1179
- "loss": 0.034,
1180
- "step": 1950
1181
- },
1182
- {
1183
- "grad_norm": 0.6386666893959045,
1184
- "learning_rate": 8.051453560801772e-05,
1185
- "loss": 0.0384,
1186
- "step": 1960
1187
- },
1188
- {
1189
- "grad_norm": 0.5290015935897827,
1190
- "learning_rate": 8.029576629825687e-05,
1191
- "loss": 0.034,
1192
- "step": 1970
1193
- },
1194
- {
1195
- "grad_norm": 0.511027455329895,
1196
- "learning_rate": 8.007607668541362e-05,
1197
- "loss": 0.0344,
1198
- "step": 1980
1199
- },
1200
- {
1201
- "grad_norm": 0.5814989805221558,
1202
- "learning_rate": 7.985547344306161e-05,
1203
- "loss": 0.0379,
1204
- "step": 1990
1205
- },
1206
- {
1207
- "grad_norm": 0.48753857612609863,
1208
- "learning_rate": 7.963396327252812e-05,
1209
- "loss": 0.0367,
1210
- "step": 2000
1211
- },
1212
- {
1213
- "grad_norm": 0.511787474155426,
1214
- "learning_rate": 7.941155290269038e-05,
1215
- "loss": 0.0353,
1216
- "step": 2010
1217
- },
1218
- {
1219
- "grad_norm": 0.5462181568145752,
1220
- "learning_rate": 7.918824908977123e-05,
1221
- "loss": 0.0335,
1222
- "step": 2020
1223
- },
1224
- {
1225
- "grad_norm": 0.42617759108543396,
1226
- "learning_rate": 7.896405861713394e-05,
1227
- "loss": 0.034,
1228
- "step": 2030
1229
- },
1230
- {
1231
- "grad_norm": 0.4321683645248413,
1232
- "learning_rate": 7.873898829507606e-05,
1233
- "loss": 0.0337,
1234
- "step": 2040
1235
- },
1236
- {
1237
- "grad_norm": 0.48458221554756165,
1238
- "learning_rate": 7.851304496062254e-05,
1239
- "loss": 0.0336,
1240
- "step": 2050
1241
- },
1242
- {
1243
- "grad_norm": 0.4516434967517853,
1244
- "learning_rate": 7.828623547731818e-05,
1245
- "loss": 0.0334,
1246
- "step": 2060
1247
- },
1248
- {
1249
- "grad_norm": 0.5588269233703613,
1250
- "learning_rate": 7.80585667350189e-05,
1251
- "loss": 0.0343,
1252
- "step": 2070
1253
- },
1254
- {
1255
- "grad_norm": 0.5814851522445679,
1256
- "learning_rate": 7.783004564968263e-05,
1257
- "loss": 0.0343,
1258
- "step": 2080
1259
- },
1260
- {
1261
- "grad_norm": 0.548805832862854,
1262
- "learning_rate": 7.760067916315921e-05,
1263
- "loss": 0.0333,
1264
- "step": 2090
1265
- },
1266
- {
1267
- "grad_norm": 0.47741568088531494,
1268
- "learning_rate": 7.737047424297941e-05,
1269
- "loss": 0.0335,
1270
- "step": 2100
1271
- },
1272
- {
1273
- "grad_norm": 0.5174888968467712,
1274
- "learning_rate": 7.713943788214337e-05,
1275
- "loss": 0.0333,
1276
- "step": 2110
1277
- },
1278
- {
1279
- "grad_norm": 0.5185810923576355,
1280
- "learning_rate": 7.690757709890812e-05,
1281
- "loss": 0.0336,
1282
- "step": 2120
1283
- },
1284
- {
1285
- "grad_norm": 0.5128524303436279,
1286
- "learning_rate": 7.66748989365744e-05,
1287
- "loss": 0.0334,
1288
- "step": 2130
1289
- },
1290
- {
1291
- "grad_norm": 0.39958009123802185,
1292
- "learning_rate": 7.644141046327271e-05,
1293
- "loss": 0.0312,
1294
- "step": 2140
1295
- },
1296
- {
1297
- "grad_norm": 0.49459007382392883,
1298
- "learning_rate": 7.620711877174866e-05,
1299
- "loss": 0.033,
1300
- "step": 2150
1301
- },
1302
- {
1303
- "grad_norm": 0.5272864699363708,
1304
- "learning_rate": 7.597203097914732e-05,
1305
- "loss": 0.0342,
1306
- "step": 2160
1307
- },
1308
- {
1309
- "grad_norm": 0.5177196860313416,
1310
- "learning_rate": 7.573615422679726e-05,
1311
- "loss": 0.0352,
1312
- "step": 2170
1313
- },
1314
- {
1315
- "grad_norm": 0.4378889203071594,
1316
- "learning_rate": 7.549949567999345e-05,
1317
- "loss": 0.0313,
1318
- "step": 2180
1319
- },
1320
- {
1321
- "grad_norm": 0.47708237171173096,
1322
- "learning_rate": 7.526206252777968e-05,
1323
- "loss": 0.0346,
1324
- "step": 2190
1325
- },
1326
- {
1327
- "grad_norm": 0.6027482151985168,
1328
- "learning_rate": 7.50238619827301e-05,
1329
- "loss": 0.0337,
1330
- "step": 2200
1331
- },
1332
- {
1333
- "grad_norm": 0.4342920780181885,
1334
- "learning_rate": 7.478490128073022e-05,
1335
- "loss": 0.0335,
1336
- "step": 2210
1337
- },
1338
- {
1339
- "grad_norm": 0.531221866607666,
1340
- "learning_rate": 7.454518768075704e-05,
1341
- "loss": 0.0334,
1342
- "step": 2220
1343
- },
1344
- {
1345
- "grad_norm": 0.42581605911254883,
1346
- "learning_rate": 7.430472846465856e-05,
1347
- "loss": 0.0322,
1348
- "step": 2230
1349
- },
1350
- {
1351
- "grad_norm": 0.42176100611686707,
1352
- "learning_rate": 7.406353093693253e-05,
1353
- "loss": 0.0325,
1354
- "step": 2240
1355
- },
1356
- {
1357
- "grad_norm": 0.5013089179992676,
1358
- "learning_rate": 7.382160242450469e-05,
1359
- "loss": 0.033,
1360
- "step": 2250
1361
- },
1362
- {
1363
- "grad_norm": 0.4330039918422699,
1364
- "learning_rate": 7.357895027650598e-05,
1365
- "loss": 0.0312,
1366
- "step": 2260
1367
- },
1368
- {
1369
- "grad_norm": 0.4245281219482422,
1370
- "learning_rate": 7.333558186404958e-05,
1371
- "loss": 0.0316,
1372
- "step": 2270
1373
- },
1374
- {
1375
- "grad_norm": 0.5211013555526733,
1376
- "learning_rate": 7.309150458000668e-05,
1377
- "loss": 0.0325,
1378
- "step": 2280
1379
- },
1380
- {
1381
- "grad_norm": 0.6483867168426514,
1382
- "learning_rate": 7.284672583878219e-05,
1383
- "loss": 0.0326,
1384
- "step": 2290
1385
- },
1386
- {
1387
- "grad_norm": 0.4503920376300812,
1388
- "learning_rate": 7.260125307608929e-05,
1389
- "loss": 0.0337,
1390
- "step": 2300
1391
- },
1392
- {
1393
- "grad_norm": 0.40689945220947266,
1394
- "learning_rate": 7.235509374872373e-05,
1395
- "loss": 0.033,
1396
- "step": 2310
1397
- },
1398
- {
1399
- "grad_norm": 0.5707908272743225,
1400
- "learning_rate": 7.210825533433719e-05,
1401
- "loss": 0.0322,
1402
- "step": 2320
1403
- },
1404
- {
1405
- "grad_norm": 0.4584868252277374,
1406
- "learning_rate": 7.186074533121013e-05,
1407
- "loss": 0.0328,
1408
- "step": 2330
1409
- },
1410
- {
1411
- "grad_norm": 0.4298429489135742,
1412
- "learning_rate": 7.161257125802413e-05,
1413
- "loss": 0.0319,
1414
- "step": 2340
1415
- },
1416
- {
1417
- "grad_norm": 0.5181540846824646,
1418
- "learning_rate": 7.136374065363334e-05,
1419
- "loss": 0.0337,
1420
- "step": 2350
1421
- },
1422
- {
1423
- "grad_norm": 0.5237898826599121,
1424
- "learning_rate": 7.11142610768356e-05,
1425
- "loss": 0.0322,
1426
- "step": 2360
1427
- },
1428
- {
1429
- "grad_norm": 0.478541761636734,
1430
- "learning_rate": 7.086414010614276e-05,
1431
- "loss": 0.0314,
1432
- "step": 2370
1433
- },
1434
- {
1435
- "grad_norm": 0.48820534348487854,
1436
- "learning_rate": 7.061338533955043e-05,
1437
- "loss": 0.0337,
1438
- "step": 2380
1439
- },
1440
- {
1441
- "grad_norm": 0.48146507143974304,
1442
- "learning_rate": 7.036200439430725e-05,
1443
- "loss": 0.0334,
1444
- "step": 2390
1445
- },
1446
- {
1447
- "grad_norm": 0.5339111685752869,
1448
- "learning_rate": 7.01100049066835e-05,
1449
- "loss": 0.0318,
1450
- "step": 2400
1451
- },
1452
- {
1453
- "grad_norm": 0.48041751980781555,
1454
- "learning_rate": 6.985739453173903e-05,
1455
- "loss": 0.0331,
1456
- "step": 2410
1457
- },
1458
- {
1459
- "grad_norm": 0.5311410427093506,
1460
- "learning_rate": 6.960418094309085e-05,
1461
- "loss": 0.032,
1462
- "step": 2420
1463
- },
1464
- {
1465
- "grad_norm": 0.3880246877670288,
1466
- "learning_rate": 6.93503718326799e-05,
1467
- "loss": 0.0315,
1468
- "step": 2430
1469
- },
1470
- {
1471
- "grad_norm": 0.44451695680618286,
1472
- "learning_rate": 6.909597491053751e-05,
1473
- "loss": 0.0302,
1474
- "step": 2440
1475
- },
1476
- {
1477
- "grad_norm": 0.5587266683578491,
1478
- "learning_rate": 6.884099790455113e-05,
1479
- "loss": 0.0306,
1480
- "step": 2450
1481
- },
1482
- {
1483
- "grad_norm": 0.5494639873504639,
1484
- "learning_rate": 6.858544856022952e-05,
1485
- "loss": 0.0327,
1486
- "step": 2460
1487
- },
1488
- {
1489
- "grad_norm": 0.42823129892349243,
1490
- "learning_rate": 6.83293346404676e-05,
1491
- "loss": 0.0327,
1492
- "step": 2470
1493
- },
1494
- {
1495
- "grad_norm": 0.4961199462413788,
1496
- "learning_rate": 6.80726639253105e-05,
1497
- "loss": 0.0343,
1498
- "step": 2480
1499
- },
1500
- {
1501
- "grad_norm": 0.5614944696426392,
1502
- "learning_rate": 6.781544421171732e-05,
1503
- "loss": 0.0327,
1504
- "step": 2490
1505
- },
1506
- {
1507
- "grad_norm": 0.45186933875083923,
1508
- "learning_rate": 6.755768331332424e-05,
1509
- "loss": 0.0335,
1510
- "step": 2500
1511
- },
1512
- {
1513
- "grad_norm": 0.4534369111061096,
1514
- "learning_rate": 6.729938906020713e-05,
1515
- "loss": 0.0327,
1516
- "step": 2510
1517
- },
1518
- {
1519
- "grad_norm": 0.3477000892162323,
1520
- "learning_rate": 6.704056929864376e-05,
1521
- "loss": 0.0306,
1522
- "step": 2520
1523
- },
1524
- {
1525
- "grad_norm": 0.43540525436401367,
1526
- "learning_rate": 6.67812318908754e-05,
1527
- "loss": 0.0298,
1528
- "step": 2530
1529
- },
1530
- {
1531
- "grad_norm": 0.3721942901611328,
1532
- "learning_rate": 6.6521384714868e-05,
1533
- "loss": 0.029,
1534
- "step": 2540
1535
- },
1536
- {
1537
- "grad_norm": 0.5104391574859619,
1538
- "learning_rate": 6.626103566407295e-05,
1539
- "loss": 0.0303,
1540
- "step": 2550
1541
- },
1542
- {
1543
- "grad_norm": 0.38681259751319885,
1544
- "learning_rate": 6.600019264718713e-05,
1545
- "loss": 0.0301,
1546
- "step": 2560
1547
- },
1548
- {
1549
- "grad_norm": 0.4433761239051819,
1550
- "learning_rate": 6.573886358791285e-05,
1551
- "loss": 0.0319,
1552
- "step": 2570
1553
- },
1554
- {
1555
- "grad_norm": 0.413533478975296,
1556
- "learning_rate": 6.547705642471703e-05,
1557
- "loss": 0.0309,
1558
- "step": 2580
1559
- },
1560
- {
1561
- "grad_norm": 0.3686458170413971,
1562
- "learning_rate": 6.521477911059008e-05,
1563
- "loss": 0.0282,
1564
- "step": 2590
1565
- },
1566
- {
1567
- "grad_norm": 0.38820889592170715,
1568
- "learning_rate": 6.495203961280434e-05,
1569
- "loss": 0.0301,
1570
- "step": 2600
1571
- },
1572
- {
1573
- "grad_norm": 0.4620518982410431,
1574
- "learning_rate": 6.468884591267204e-05,
1575
- "loss": 0.0302,
1576
- "step": 2610
1577
- },
1578
- {
1579
- "grad_norm": 0.5960747599601746,
1580
- "learning_rate": 6.44252060053028e-05,
1581
- "loss": 0.03,
1582
- "step": 2620
1583
- },
1584
- {
1585
- "grad_norm": 0.6030673980712891,
1586
- "learning_rate": 6.416112789936086e-05,
1587
- "loss": 0.0312,
1588
- "step": 2630
1589
- },
1590
- {
1591
- "grad_norm": 0.5022603273391724,
1592
- "learning_rate": 6.389661961682173e-05,
1593
- "loss": 0.0302,
1594
- "step": 2640
1595
- },
1596
- {
1597
- "grad_norm": 0.4202682375907898,
1598
- "learning_rate": 6.363168919272846e-05,
1599
- "loss": 0.0295,
1600
- "step": 2650
1601
- },
1602
- {
1603
- "grad_norm": 0.40000995993614197,
1604
- "learning_rate": 6.336634467494768e-05,
1605
- "loss": 0.0286,
1606
- "step": 2660
1607
- },
1608
- {
1609
- "grad_norm": 0.49214911460876465,
1610
- "learning_rate": 6.310059412392505e-05,
1611
- "loss": 0.0308,
1612
- "step": 2670
1613
- },
1614
- {
1615
- "grad_norm": 0.4698426127433777,
1616
- "learning_rate": 6.283444561244042e-05,
1617
- "loss": 0.0291,
1618
- "step": 2680
1619
- },
1620
- {
1621
- "grad_norm": 0.48712682723999023,
1622
- "learning_rate": 6.256790722536251e-05,
1623
- "loss": 0.0288,
1624
- "step": 2690
1625
- },
1626
- {
1627
- "grad_norm": 0.46532660722732544,
1628
- "learning_rate": 6.230098705940354e-05,
1629
- "loss": 0.0292,
1630
- "step": 2700
1631
- },
1632
- {
1633
- "grad_norm": 0.422234445810318,
1634
- "learning_rate": 6.203369322287306e-05,
1635
- "loss": 0.0295,
1636
- "step": 2710
1637
- },
1638
- {
1639
- "grad_norm": 0.3877371549606323,
1640
- "learning_rate": 6.17660338354317e-05,
1641
- "loss": 0.0281,
1642
- "step": 2720
1643
- },
1644
- {
1645
- "grad_norm": 0.4343086779117584,
1646
- "learning_rate": 6.149801702784456e-05,
1647
- "loss": 0.0279,
1648
- "step": 2730
1649
- },
1650
- {
1651
- "grad_norm": 0.4151124954223633,
1652
- "learning_rate": 6.122965094173424e-05,
1653
- "loss": 0.0288,
1654
- "step": 2740
1655
- },
1656
- {
1657
- "grad_norm": 0.5097172260284424,
1658
- "learning_rate": 6.0960943729333374e-05,
1659
- "loss": 0.0306,
1660
- "step": 2750
1661
- },
1662
- {
1663
- "grad_norm": 0.5084396600723267,
1664
- "learning_rate": 6.069190355323717e-05,
1665
- "loss": 0.0284,
1666
- "step": 2760
1667
- },
1668
- {
1669
- "grad_norm": 0.46339768171310425,
1670
- "learning_rate": 6.042253858615532e-05,
1671
- "loss": 0.0284,
1672
- "step": 2770
1673
- },
1674
- {
1675
- "grad_norm": 0.4735918343067169,
1676
- "learning_rate": 6.015285701066382e-05,
1677
- "loss": 0.0304,
1678
- "step": 2780
1679
- },
1680
- {
1681
- "grad_norm": 0.4199438691139221,
1682
- "learning_rate": 5.988286701895631e-05,
1683
- "loss": 0.0321,
1684
- "step": 2790
1685
- },
1686
- {
1687
- "grad_norm": 0.5070331692695618,
1688
- "learning_rate": 5.961257681259535e-05,
1689
- "loss": 0.0313,
1690
- "step": 2800
1691
- },
1692
- {
1693
- "grad_norm": 0.419735848903656,
1694
- "learning_rate": 5.934199460226317e-05,
1695
- "loss": 0.0306,
1696
- "step": 2810
1697
- },
1698
- {
1699
- "grad_norm": 0.4207720160484314,
1700
- "learning_rate": 5.9071128607512285e-05,
1701
- "loss": 0.0295,
1702
- "step": 2820
1703
- },
1704
- {
1705
- "grad_norm": 0.3231958746910095,
1706
- "learning_rate": 5.8799987056515804e-05,
1707
- "loss": 0.0288,
1708
- "step": 2830
1709
- },
1710
- {
1711
- "grad_norm": 0.31990593671798706,
1712
- "learning_rate": 5.8528578185817514e-05,
1713
- "loss": 0.0296,
1714
- "step": 2840
1715
- },
1716
- {
1717
- "grad_norm": 0.41715797781944275,
1718
- "learning_rate": 5.825691024008162e-05,
1719
- "loss": 0.0287,
1720
- "step": 2850
1721
- },
1722
- {
1723
- "grad_norm": 0.3970031142234802,
1724
- "learning_rate": 5.798499147184233e-05,
1725
- "loss": 0.0286,
1726
- "step": 2860
1727
- },
1728
- {
1729
- "grad_norm": 0.3385803699493408,
1730
- "learning_rate": 5.771283014125317e-05,
1731
- "loss": 0.0309,
1732
- "step": 2870
1733
- },
1734
- {
1735
- "grad_norm": 0.42199286818504333,
1736
- "learning_rate": 5.7440434515836064e-05,
1737
- "loss": 0.0283,
1738
- "step": 2880
1739
- },
1740
- {
1741
- "grad_norm": 0.4184113144874573,
1742
- "learning_rate": 5.7167812870230094e-05,
1743
- "loss": 0.0281,
1744
- "step": 2890
1745
- },
1746
- {
1747
- "grad_norm": 0.4537411630153656,
1748
- "learning_rate": 5.689497348594035e-05,
1749
- "loss": 0.0282,
1750
- "step": 2900
1751
- },
1752
- {
1753
- "grad_norm": 0.3758944869041443,
1754
- "learning_rate": 5.662192465108613e-05,
1755
- "loss": 0.0291,
1756
- "step": 2910
1757
- },
1758
- {
1759
- "grad_norm": 0.43658238649368286,
1760
- "learning_rate": 5.634867466014932e-05,
1761
- "loss": 0.0276,
1762
- "step": 2920
1763
- },
1764
- {
1765
- "grad_norm": 0.435166597366333,
1766
- "learning_rate": 5.607523181372234e-05,
1767
- "loss": 0.029,
1768
- "step": 2930
1769
- },
1770
- {
1771
- "grad_norm": 0.39249086380004883,
1772
- "learning_rate": 5.5801604418256117e-05,
1773
- "loss": 0.0281,
1774
- "step": 2940
1775
- },
1776
- {
1777
- "grad_norm": 0.38982075452804565,
1778
- "learning_rate": 5.552780078580756e-05,
1779
- "loss": 0.0267,
1780
- "step": 2950
1781
- },
1782
- {
1783
- "grad_norm": 0.397555410861969,
1784
- "learning_rate": 5.525382923378728e-05,
1785
- "loss": 0.028,
1786
- "step": 2960
1787
- },
1788
- {
1789
- "grad_norm": 0.4088326692581177,
1790
- "learning_rate": 5.49796980847068e-05,
1791
- "loss": 0.0299,
1792
- "step": 2970
1793
- },
1794
- {
1795
- "grad_norm": 0.45356932282447815,
1796
- "learning_rate": 5.470541566592573e-05,
1797
- "loss": 0.0296,
1798
- "step": 2980
1799
- },
1800
- {
1801
- "grad_norm": 0.3560492694377899,
1802
- "learning_rate": 5.443099030939887e-05,
1803
- "loss": 0.0278,
1804
- "step": 2990
1805
- },
1806
- {
1807
- "grad_norm": 0.35022470355033875,
1808
- "learning_rate": 5.415643035142309e-05,
1809
- "loss": 0.0275,
1810
- "step": 3000
1811
- },
1812
- {
1813
- "grad_norm": 0.40406009554862976,
1814
- "learning_rate": 5.3881744132384104e-05,
1815
- "loss": 0.0286,
1816
- "step": 3010
1817
- },
1818
- {
1819
- "grad_norm": 0.3887314796447754,
1820
- "learning_rate": 5.360693999650303e-05,
1821
- "loss": 0.0278,
1822
- "step": 3020
1823
- },
1824
- {
1825
- "grad_norm": 0.44561782479286194,
1826
- "learning_rate": 5.3332026291583016e-05,
1827
- "loss": 0.0274,
1828
- "step": 3030
1829
- },
1830
- {
1831
- "grad_norm": 0.4164481461048126,
1832
- "learning_rate": 5.305701136875566e-05,
1833
- "loss": 0.0292,
1834
- "step": 3040
1835
- },
1836
- {
1837
- "grad_norm": 0.3928636610507965,
1838
- "learning_rate": 5.278190358222721e-05,
1839
- "loss": 0.0281,
1840
- "step": 3050
1841
- },
1842
- {
1843
- "grad_norm": 0.4332394003868103,
1844
- "learning_rate": 5.25067112890249e-05,
1845
- "loss": 0.0286,
1846
- "step": 3060
1847
- },
1848
- {
1849
- "grad_norm": 0.33109888434410095,
1850
- "learning_rate": 5.2231442848743064e-05,
1851
- "loss": 0.0284,
1852
- "step": 3070
1853
- },
1854
- {
1855
- "grad_norm": 0.44803181290626526,
1856
- "learning_rate": 5.1956106623289145e-05,
1857
- "loss": 0.0261,
1858
- "step": 3080
1859
- },
1860
- {
1861
- "grad_norm": 0.39142924547195435,
1862
- "learning_rate": 5.168071097662972e-05,
1863
- "loss": 0.0261,
1864
- "step": 3090
1865
- },
1866
- {
1867
- "grad_norm": 0.532752513885498,
1868
- "learning_rate": 5.1405264274536445e-05,
1869
- "loss": 0.0278,
1870
- "step": 3100
1871
- },
1872
- {
1873
- "grad_norm": 0.4373730421066284,
1874
- "learning_rate": 5.112977488433188e-05,
1875
- "loss": 0.0281,
1876
- "step": 3110
1877
- },
1878
- {
1879
- "grad_norm": 0.395621120929718,
1880
- "learning_rate": 5.085425117463533e-05,
1881
- "loss": 0.027,
1882
- "step": 3120
1883
- },
1884
- {
1885
- "grad_norm": 0.43923935294151306,
1886
- "learning_rate": 5.057870151510864e-05,
1887
- "loss": 0.0264,
1888
- "step": 3130
1889
- },
1890
- {
1891
- "grad_norm": 0.37473028898239136,
1892
- "learning_rate": 5.030313427620197e-05,
1893
- "loss": 0.0261,
1894
- "step": 3140
1895
- },
1896
- {
1897
- "grad_norm": 0.44424837827682495,
1898
- "learning_rate": 5.0027557828899426e-05,
1899
- "loss": 0.0269,
1900
- "step": 3150
1901
- },
1902
- {
1903
- "grad_norm": 0.40588265657424927,
1904
- "learning_rate": 4.975198054446492e-05,
1905
- "loss": 0.0262,
1906
- "step": 3160
1907
- },
1908
- {
1909
- "grad_norm": 0.4514497220516205,
1910
- "learning_rate": 4.947641079418773e-05,
1911
- "loss": 0.0267,
1912
- "step": 3170
1913
- },
1914
- {
1915
- "grad_norm": 0.5240787267684937,
1916
- "learning_rate": 4.920085694912828e-05,
1917
- "loss": 0.0287,
1918
- "step": 3180
1919
- },
1920
- {
1921
- "grad_norm": 0.37988799810409546,
1922
- "learning_rate": 4.892532737986387e-05,
1923
- "loss": 0.0265,
1924
- "step": 3190
1925
- },
1926
- {
1927
- "grad_norm": 0.376968115568161,
1928
- "learning_rate": 4.864983045623434e-05,
1929
- "loss": 0.0292,
1930
- "step": 3200
1931
- },
1932
- {
1933
- "grad_norm": 0.3950600326061249,
1934
- "learning_rate": 4.837437454708784e-05,
1935
- "loss": 0.0279,
1936
- "step": 3210
1937
- },
1938
- {
1939
- "grad_norm": 0.37644240260124207,
1940
- "learning_rate": 4.809896802002662e-05,
1941
- "loss": 0.0259,
1942
- "step": 3220
1943
- },
1944
- {
1945
- "grad_norm": 0.39114123582839966,
1946
- "learning_rate": 4.7823619241152854e-05,
1947
- "loss": 0.0256,
1948
- "step": 3230
1949
- },
1950
- {
1951
- "grad_norm": 0.2755090594291687,
1952
- "learning_rate": 4.754833657481445e-05,
1953
- "loss": 0.0248,
1954
- "step": 3240
1955
- },
1956
- {
1957
- "grad_norm": 0.3851339519023895,
1958
- "learning_rate": 4.7273128383351015e-05,
1959
- "loss": 0.0274,
1960
- "step": 3250
1961
- },
1962
- {
1963
- "grad_norm": 0.4715338349342346,
1964
- "learning_rate": 4.699800302683981e-05,
1965
- "loss": 0.0273,
1966
- "step": 3260
1967
- },
1968
- {
1969
- "grad_norm": 0.30057212710380554,
1970
- "learning_rate": 4.6722968862841806e-05,
1971
- "loss": 0.0251,
1972
- "step": 3270
1973
- },
1974
- {
1975
- "grad_norm": 0.3959311544895172,
1976
- "learning_rate": 4.6448034246147754e-05,
1977
- "loss": 0.0267,
1978
- "step": 3280
1979
- },
1980
- {
1981
- "grad_norm": 0.35625332593917847,
1982
- "learning_rate": 4.6173207528524476e-05,
1983
- "loss": 0.0255,
1984
- "step": 3290
1985
- },
1986
- {
1987
- "grad_norm": 0.35969534516334534,
1988
- "learning_rate": 4.58984970584611e-05,
1989
- "loss": 0.0263,
1990
- "step": 3300
1991
- },
1992
- {
1993
- "grad_norm": 0.5300602316856384,
1994
- "learning_rate": 4.562391118091544e-05,
1995
- "loss": 0.0279,
1996
- "step": 3310
1997
- },
1998
- {
1999
- "grad_norm": 0.5815778374671936,
2000
- "learning_rate": 4.534945823706056e-05,
2001
- "loss": 0.0291,
2002
- "step": 3320
2003
- },
2004
- {
2005
- "grad_norm": 0.41548240184783936,
2006
- "learning_rate": 4.507514656403137e-05,
2007
- "loss": 0.0273,
2008
- "step": 3330
2009
- },
2010
- {
2011
- "grad_norm": 0.38885802030563354,
2012
- "learning_rate": 4.480098449467132e-05,
2013
- "loss": 0.0265,
2014
- "step": 3340
2015
- },
2016
- {
2017
- "grad_norm": 0.3587275445461273,
2018
- "learning_rate": 4.452698035727929e-05,
2019
- "loss": 0.0255,
2020
- "step": 3350
2021
- },
2022
- {
2023
- "grad_norm": 0.42804276943206787,
2024
- "learning_rate": 4.425314247535668e-05,
2025
- "loss": 0.0272,
2026
- "step": 3360
2027
- },
2028
- {
2029
- "grad_norm": 0.37650251388549805,
2030
- "learning_rate": 4.3979479167354477e-05,
2031
- "loss": 0.0263,
2032
- "step": 3370
2033
- },
2034
- {
2035
- "grad_norm": 0.3508603274822235,
2036
- "learning_rate": 4.370599874642055e-05,
2037
- "loss": 0.0288,
2038
- "step": 3380
2039
- },
2040
- {
2041
- "grad_norm": 0.3932643532752991,
2042
- "learning_rate": 4.3432709520147205e-05,
2043
- "loss": 0.0271,
2044
- "step": 3390
2045
- },
2046
- {
2047
- "grad_norm": 0.3967253863811493,
2048
- "learning_rate": 4.315961979031875e-05,
2049
- "loss": 0.0262,
2050
- "step": 3400
2051
- },
2052
- {
2053
- "grad_norm": 0.34985652565956116,
2054
- "learning_rate": 4.2886737852659325e-05,
2055
- "loss": 0.0266,
2056
- "step": 3410
2057
- },
2058
- {
2059
- "grad_norm": 0.4587637484073639,
2060
- "learning_rate": 4.261407199658093e-05,
2061
- "loss": 0.0261,
2062
- "step": 3420
2063
- },
2064
- {
2065
- "grad_norm": 0.40246912837028503,
2066
- "learning_rate": 4.234163050493158e-05,
2067
- "loss": 0.0248,
2068
- "step": 3430
2069
- },
2070
- {
2071
- "grad_norm": 0.4552555978298187,
2072
- "learning_rate": 4.2069421653743706e-05,
2073
- "loss": 0.0264,
2074
- "step": 3440
2075
- },
2076
- {
2077
- "grad_norm": 0.31220653653144836,
2078
- "learning_rate": 4.179745371198276e-05,
2079
- "loss": 0.0253,
2080
- "step": 3450
2081
- },
2082
- {
2083
- "grad_norm": 0.38440898060798645,
2084
- "learning_rate": 4.1525734941296026e-05,
2085
- "loss": 0.0263,
2086
- "step": 3460
2087
- },
2088
- {
2089
- "grad_norm": 0.3631040155887604,
2090
- "learning_rate": 4.125427359576162e-05,
2091
- "loss": 0.0254,
2092
- "step": 3470
2093
- },
2094
- {
2095
- "grad_norm": 0.41102805733680725,
2096
- "learning_rate": 4.0983077921637815e-05,
2097
- "loss": 0.0253,
2098
- "step": 3480
2099
- },
2100
- {
2101
- "grad_norm": 0.38240450620651245,
2102
- "learning_rate": 4.07121561571125e-05,
2103
- "loss": 0.0255,
2104
- "step": 3490
2105
- },
2106
- {
2107
- "grad_norm": 0.3399236798286438,
2108
- "learning_rate": 4.044151653205292e-05,
2109
- "loss": 0.0238,
2110
- "step": 3500
2111
- },
2112
- {
2113
- "grad_norm": 0.37793099880218506,
2114
- "learning_rate": 4.0171167267755696e-05,
2115
- "loss": 0.0248,
2116
- "step": 3510
2117
- },
2118
- {
2119
- "grad_norm": 0.47253522276878357,
2120
- "learning_rate": 3.9901116576697083e-05,
2121
- "loss": 0.0243,
2122
- "step": 3520
2123
- },
2124
- {
2125
- "grad_norm": 0.42441272735595703,
2126
- "learning_rate": 3.963137266228349e-05,
2127
- "loss": 0.0239,
2128
- "step": 3530
2129
- },
2130
- {
2131
- "grad_norm": 0.378666490316391,
2132
- "learning_rate": 3.93619437186023e-05,
2133
- "loss": 0.0265,
2134
- "step": 3540
2135
- },
2136
- {
2137
- "grad_norm": 0.41540664434432983,
2138
- "learning_rate": 3.9092837930172884e-05,
2139
- "loss": 0.0253,
2140
- "step": 3550
2141
- },
2142
- {
2143
- "grad_norm": 0.3746972680091858,
2144
- "learning_rate": 3.8824063471698105e-05,
2145
- "loss": 0.0251,
2146
- "step": 3560
2147
- },
2148
- {
2149
- "grad_norm": 0.3388042151927948,
2150
- "learning_rate": 3.855562850781589e-05,
2151
- "loss": 0.0249,
2152
- "step": 3570
2153
- },
2154
- {
2155
- "grad_norm": 0.32412514090538025,
2156
- "learning_rate": 3.828754119285123e-05,
2157
- "loss": 0.0247,
2158
- "step": 3580
2159
- },
2160
- {
2161
- "grad_norm": 0.32984834909439087,
2162
- "learning_rate": 3.801980967056851e-05,
2163
- "loss": 0.0257,
2164
- "step": 3590
2165
- },
2166
- {
2167
- "grad_norm": 0.3018534481525421,
2168
- "learning_rate": 3.77524420739241e-05,
2169
- "loss": 0.0254,
2170
- "step": 3600
2171
- },
2172
- {
2173
- "grad_norm": 0.34701016545295715,
2174
- "learning_rate": 3.748544652481927e-05,
2175
- "loss": 0.024,
2176
- "step": 3610
2177
- },
2178
- {
2179
- "grad_norm": 0.4032657742500305,
2180
- "learning_rate": 3.721883113385353e-05,
2181
- "loss": 0.0268,
2182
- "step": 3620
2183
- },
2184
- {
2185
- "grad_norm": 0.3056441843509674,
2186
- "learning_rate": 3.695260400007819e-05,
2187
- "loss": 0.023,
2188
- "step": 3630
2189
- },
2190
- {
2191
- "grad_norm": 0.34800854325294495,
2192
- "learning_rate": 3.6686773210750385e-05,
2193
- "loss": 0.0253,
2194
- "step": 3640
2195
- },
2196
- {
2197
- "grad_norm": 0.3697214424610138,
2198
- "learning_rate": 3.642134684108737e-05,
2199
- "loss": 0.0246,
2200
- "step": 3650
2201
- },
2202
- {
2203
- "grad_norm": 0.31739336252212524,
2204
- "learning_rate": 3.615633295402123e-05,
2205
- "loss": 0.0254,
2206
- "step": 3660
2207
- },
2208
- {
2209
- "grad_norm": 0.3576298952102661,
2210
- "learning_rate": 3.5891739599953945e-05,
2211
- "loss": 0.0231,
2212
- "step": 3670
2213
- },
2214
- {
2215
- "grad_norm": 0.4195830225944519,
2216
- "learning_rate": 3.5627574816512846e-05,
2217
- "loss": 0.0271,
2218
- "step": 3680
2219
- },
2220
- {
2221
- "grad_norm": 0.38255468010902405,
2222
- "learning_rate": 3.536384662830648e-05,
2223
- "loss": 0.0263,
2224
- "step": 3690
2225
- },
2226
- {
2227
- "grad_norm": 0.31736278533935547,
2228
- "learning_rate": 3.5100563046680764e-05,
2229
- "loss": 0.0243,
2230
- "step": 3700
2231
- },
2232
- {
2233
- "grad_norm": 0.38382217288017273,
2234
- "learning_rate": 3.483773206947572e-05,
2235
- "loss": 0.0233,
2236
- "step": 3710
2237
- },
2238
- {
2239
- "grad_norm": 0.37553671002388,
2240
- "learning_rate": 3.457536168078247e-05,
2241
- "loss": 0.0233,
2242
- "step": 3720
2243
- },
2244
- {
2245
- "grad_norm": 0.45702260732650757,
2246
- "learning_rate": 3.431345985070067e-05,
2247
- "loss": 0.0248,
2248
- "step": 3730
2249
- },
2250
- {
2251
- "grad_norm": 0.30532407760620117,
2252
- "learning_rate": 3.40520345350965e-05,
2253
- "loss": 0.0232,
2254
- "step": 3740
2255
- },
2256
- {
2257
- "grad_norm": 0.3560055196285248,
2258
- "learning_rate": 3.379109367536089e-05,
2259
- "loss": 0.0237,
2260
- "step": 3750
2261
- },
2262
- {
2263
- "grad_norm": 0.3362725079059601,
2264
- "learning_rate": 3.3530645198168295e-05,
2265
- "loss": 0.0243,
2266
- "step": 3760
2267
- },
2268
- {
2269
- "grad_norm": 0.32288989424705505,
2270
- "learning_rate": 3.327069701523595e-05,
2271
- "loss": 0.0232,
2272
- "step": 3770
2273
- },
2274
- {
2275
- "grad_norm": 0.35202544927597046,
2276
- "learning_rate": 3.301125702308353e-05,
2277
- "loss": 0.0248,
2278
- "step": 3780
2279
- },
2280
- {
2281
- "grad_norm": 0.3243369460105896,
2282
- "learning_rate": 3.275233310279321e-05,
2283
- "loss": 0.023,
2284
- "step": 3790
2285
- },
2286
- {
2287
- "grad_norm": 0.3195122480392456,
2288
- "learning_rate": 3.249393311977037e-05,
2289
- "loss": 0.024,
2290
- "step": 3800
2291
- },
2292
- {
2293
- "grad_norm": 0.26877087354660034,
2294
- "learning_rate": 3.223606492350451e-05,
2295
- "loss": 0.0232,
2296
- "step": 3810
2297
- },
2298
- {
2299
- "grad_norm": 0.36210641264915466,
2300
- "learning_rate": 3.197873634733096e-05,
2301
- "loss": 0.0241,
2302
- "step": 3820
2303
- },
2304
- {
2305
- "grad_norm": 0.3320061266422272,
2306
- "learning_rate": 3.172195520819285e-05,
2307
- "loss": 0.0247,
2308
- "step": 3830
2309
- },
2310
- {
2311
- "grad_norm": 0.44787928462028503,
2312
- "learning_rate": 3.146572930640362e-05,
2313
- "loss": 0.0233,
2314
- "step": 3840
2315
- },
2316
- {
2317
- "grad_norm": 0.3908366858959198,
2318
- "learning_rate": 3.121006642541014e-05,
2319
- "loss": 0.0246,
2320
- "step": 3850
2321
- },
2322
- {
2323
- "grad_norm": 0.3663296103477478,
2324
- "learning_rate": 3.095497433155626e-05,
2325
- "loss": 0.0236,
2326
- "step": 3860
2327
- },
2328
- {
2329
- "grad_norm": 0.36651811003685,
2330
- "learning_rate": 3.070046077384682e-05,
2331
- "loss": 0.0234,
2332
- "step": 3870
2333
- },
2334
- {
2335
- "grad_norm": 0.2849855422973633,
2336
- "learning_rate": 3.0446533483712304e-05,
2337
- "loss": 0.0231,
2338
- "step": 3880
2339
- },
2340
- {
2341
- "grad_norm": 0.30322885513305664,
2342
- "learning_rate": 3.0193200174774038e-05,
2343
- "loss": 0.0258,
2344
- "step": 3890
2345
- },
2346
- {
2347
- "grad_norm": 0.34874120354652405,
2348
- "learning_rate": 2.994046854260974e-05,
2349
- "loss": 0.0217,
2350
- "step": 3900
2351
- },
2352
- {
2353
- "grad_norm": 0.3394639790058136,
2354
- "learning_rate": 2.9688346264519866e-05,
2355
- "loss": 0.0218,
2356
- "step": 3910
2357
- },
2358
- {
2359
- "grad_norm": 0.2971331775188446,
2360
- "learning_rate": 2.943684099929436e-05,
2361
- "loss": 0.0235,
2362
- "step": 3920
2363
- },
2364
- {
2365
- "grad_norm": 0.3201124370098114,
2366
- "learning_rate": 2.918596038697995e-05,
2367
- "loss": 0.0233,
2368
- "step": 3930
2369
- },
2370
- {
2371
- "grad_norm": 0.3573104739189148,
2372
- "learning_rate": 2.8935712048648112e-05,
2373
- "loss": 0.0222,
2374
- "step": 3940
2375
- },
2376
- {
2377
- "grad_norm": 0.3573231101036072,
2378
- "learning_rate": 2.8686103586163626e-05,
2379
- "loss": 0.0232,
2380
- "step": 3950
2381
- },
2382
- {
2383
- "grad_norm": 0.3325694799423218,
2384
- "learning_rate": 2.843714258195346e-05,
2385
- "loss": 0.0229,
2386
- "step": 3960
2387
- },
2388
- {
2389
- "grad_norm": 0.349811315536499,
2390
- "learning_rate": 2.8188836598776662e-05,
2391
- "loss": 0.0236,
2392
- "step": 3970
2393
- },
2394
- {
2395
- "grad_norm": 0.3447701334953308,
2396
- "learning_rate": 2.7941193179494484e-05,
2397
- "loss": 0.0236,
2398
- "step": 3980
2399
- },
2400
- {
2401
- "grad_norm": 0.3687022030353546,
2402
- "learning_rate": 2.7694219846841262e-05,
2403
- "loss": 0.0249,
2404
- "step": 3990
2405
- },
2406
- {
2407
- "grad_norm": 0.3964892327785492,
2408
- "learning_rate": 2.7447924103195976e-05,
2409
- "loss": 0.0222,
2410
- "step": 4000
2411
- }
2412
- ],
2413
- "logging_steps": 10,
2414
- "max_steps": 6000,
2415
- "num_input_tokens_seen": 0,
2416
- "num_train_epochs": 9223372036854775807,
2417
- "save_steps": 2000,
2418
- "stateful_callbacks": {
2419
- "TrainerControl": {
2420
- "args": {
2421
- "should_epoch_stop": false,
2422
- "should_evaluate": false,
2423
- "should_log": false,
2424
- "should_save": true,
2425
- "should_training_stop": false
2426
- },
2427
- "attributes": {}
2428
- }
2429
- },
2430
- "total_flos": 0.0,
2431
- "train_batch_size": 128,
2432
- "trial_name": null,
2433
- "trial_params": null
2434
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-4000/training_args.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:c25b10b9ea2b48c7d392e8e72ee1e37b2d2fadcf822e76df74171586e2f74941
3
- size 7633
 
 
 
 
checkpoint-4000/wandb_config.json DELETED
@@ -1 +0,0 @@
1
- {"project": "finetune-gr00t-n1d6", "run_id": "g1_finetune"}
 
 
checkpoint-4000/zero_to_fp32.py DELETED
@@ -1,760 +0,0 @@
1
- #!/usr/bin/env python
2
-
3
- # Copyright (c) Microsoft Corporation.
4
- # SPDX-License-Identifier: Apache-2.0
5
-
6
- # DeepSpeed Team
7
-
8
- # This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
9
- # copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
10
- # the future. Once extracted, the weights don't require DeepSpeed and can be used in any
11
- # application.
12
- #
13
- # example:
14
- # python zero_to_fp32.py . output_dir/
15
- # or
16
- # python zero_to_fp32.py . output_dir/ --safe_serialization
17
-
18
- import argparse
19
- import torch
20
- import glob
21
- import math
22
- import os
23
- import re
24
- import gc
25
- import json
26
- import numpy as np
27
- from tqdm import tqdm
28
- from collections import OrderedDict
29
- from dataclasses import dataclass
30
-
31
- # while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
32
- # DeepSpeed data structures it has to be available in the current python environment.
33
- from deepspeed.utils import logger
34
- from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
35
- FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
36
- FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
37
-
38
-
39
- @dataclass
40
- class zero_model_state:
41
- buffers: dict()
42
- param_shapes: dict()
43
- shared_params: list
44
- ds_version: int
45
- frozen_param_shapes: dict()
46
- frozen_param_fragments: dict()
47
-
48
-
49
- debug = 0
50
-
51
- # load to cpu
52
- device = torch.device('cpu')
53
-
54
-
55
- def atoi(text):
56
- return int(text) if text.isdigit() else text
57
-
58
-
59
- def natural_keys(text):
60
- '''
61
- alist.sort(key=natural_keys) sorts in human order
62
- http://nedbatchelder.com/blog/200712/human_sorting.html
63
- (See Toothy's implementation in the comments)
64
- '''
65
- return [atoi(c) for c in re.split(r'(\d+)', text)]
66
-
67
-
68
- def get_model_state_file(checkpoint_dir, zero_stage):
69
- if not os.path.isdir(checkpoint_dir):
70
- raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
71
-
72
- # there should be only one file
73
- if zero_stage <= 2:
74
- file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
75
- elif zero_stage == 3:
76
- file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
77
-
78
- if not os.path.exists(file):
79
- raise FileNotFoundError(f"can't find model states file at '{file}'")
80
-
81
- return file
82
-
83
-
84
- def get_checkpoint_files(checkpoint_dir, glob_pattern):
85
- # XXX: need to test that this simple glob rule works for multi-node setup too
86
- ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
87
-
88
- if len(ckpt_files) == 0:
89
- raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
90
-
91
- return ckpt_files
92
-
93
-
94
- def get_optim_files(checkpoint_dir):
95
- return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
96
-
97
-
98
- def get_model_state_files(checkpoint_dir):
99
- return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
100
-
101
-
102
- def parse_model_states(files):
103
- zero_model_states = []
104
- for file in files:
105
- state_dict = torch.load(file, map_location=device, weights_only=False)
106
-
107
- if BUFFER_NAMES not in state_dict:
108
- raise ValueError(f"{file} is not a model state checkpoint")
109
- buffer_names = state_dict[BUFFER_NAMES]
110
- if debug:
111
- print("Found buffers:", buffer_names)
112
-
113
- # recover just the buffers while restoring them to fp32 if they were saved in fp16
114
- buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
115
- param_shapes = state_dict[PARAM_SHAPES]
116
-
117
- # collect parameters that are included in param_shapes
118
- param_names = []
119
- for s in param_shapes:
120
- for name in s.keys():
121
- param_names.append(name)
122
-
123
- # update with frozen parameters
124
- frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
125
- if frozen_param_shapes is not None:
126
- if debug:
127
- print(f"Found frozen_param_shapes: {frozen_param_shapes}")
128
- param_names += list(frozen_param_shapes.keys())
129
-
130
- # handle shared params
131
- shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
132
-
133
- ds_version = state_dict.get(DS_VERSION, None)
134
-
135
- frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
136
-
137
- z_model_state = zero_model_state(buffers=buffers,
138
- param_shapes=param_shapes,
139
- shared_params=shared_params,
140
- ds_version=ds_version,
141
- frozen_param_shapes=frozen_param_shapes,
142
- frozen_param_fragments=frozen_param_fragments)
143
- zero_model_states.append(z_model_state)
144
-
145
- return zero_model_states
146
-
147
-
148
- def parse_optim_states(files, ds_checkpoint_dir):
149
- total_files = len(files)
150
- state_dicts = []
151
- for f in tqdm(files, desc='Loading checkpoint shards'):
152
- state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
153
- # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
154
- # and also handle the case where it was already removed by another helper script
155
- state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
156
- state_dicts.append(state_dict)
157
-
158
- if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
159
- raise ValueError(f"{files[0]} is not a zero checkpoint")
160
- zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
161
- world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
162
-
163
- # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
164
- # parameters can be different from data parallelism for non-expert parameters. So we can just
165
- # use the max of the partition_count to get the dp world_size.
166
-
167
- if type(world_size) is list:
168
- world_size = max(world_size)
169
-
170
- if world_size != total_files:
171
- raise ValueError(
172
- f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
173
- "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
174
- )
175
-
176
- # the groups are named differently in each stage
177
- if zero_stage <= 2:
178
- fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
179
- elif zero_stage == 3:
180
- fp32_groups_key = FP32_FLAT_GROUPS
181
- else:
182
- raise ValueError(f"unknown zero stage {zero_stage}")
183
-
184
- fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
185
- return zero_stage, world_size, fp32_flat_groups
186
-
187
-
188
- def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
189
- """
190
- Returns fp32 state_dict reconstructed from ds checkpoint
191
-
192
- Args:
193
- - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
194
-
195
- """
196
- print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
197
-
198
- optim_files = get_optim_files(ds_checkpoint_dir)
199
- zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
200
- print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
201
-
202
- model_files = get_model_state_files(ds_checkpoint_dir)
203
-
204
- zero_model_states = parse_model_states(model_files)
205
- print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
206
-
207
- if zero_stage <= 2:
208
- return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
209
- exclude_frozen_parameters)
210
- elif zero_stage == 3:
211
- return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
212
- exclude_frozen_parameters)
213
-
214
-
215
- def _zero2_merge_frozen_params(state_dict, zero_model_states):
216
- if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
217
- return
218
-
219
- frozen_param_shapes = zero_model_states[0].frozen_param_shapes
220
- frozen_param_fragments = zero_model_states[0].frozen_param_fragments
221
-
222
- if debug:
223
- num_elem = sum(s.numel() for s in frozen_param_shapes.values())
224
- print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
225
-
226
- wanted_params = len(frozen_param_shapes)
227
- wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
228
- avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
229
- print(f'Frozen params: Have {avail_numel} numels to process.')
230
- print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
231
-
232
- total_params = 0
233
- total_numel = 0
234
- for name, shape in frozen_param_shapes.items():
235
- total_params += 1
236
- unpartitioned_numel = shape.numel()
237
- total_numel += unpartitioned_numel
238
-
239
- state_dict[name] = frozen_param_fragments[name]
240
-
241
- if debug:
242
- print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
243
-
244
- print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
245
-
246
-
247
- def _has_callable(obj, fn):
248
- attr = getattr(obj, fn, None)
249
- return callable(attr)
250
-
251
-
252
- def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
253
- param_shapes = zero_model_states[0].param_shapes
254
-
255
- # Reconstruction protocol:
256
- #
257
- # XXX: document this
258
-
259
- if debug:
260
- for i in range(world_size):
261
- for j in range(len(fp32_flat_groups[0])):
262
- print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
263
-
264
- # XXX: memory usage doubles here (zero2)
265
- num_param_groups = len(fp32_flat_groups[0])
266
- merged_single_partition_of_fp32_groups = []
267
- for i in range(num_param_groups):
268
- merged_partitions = [sd[i] for sd in fp32_flat_groups]
269
- full_single_fp32_vector = torch.cat(merged_partitions, 0)
270
- merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
271
- avail_numel = sum(
272
- [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
273
-
274
- if debug:
275
- wanted_params = sum([len(shapes) for shapes in param_shapes])
276
- wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
277
- # not asserting if there is a mismatch due to possible padding
278
- print(f"Have {avail_numel} numels to process.")
279
- print(f"Need {wanted_numel} numels in {wanted_params} params.")
280
-
281
- # params
282
- # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
283
- # out-of-core computing solution
284
- total_numel = 0
285
- total_params = 0
286
- for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
287
- offset = 0
288
- avail_numel = full_single_fp32_vector.numel()
289
- for name, shape in shapes.items():
290
-
291
- unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
292
- total_numel += unpartitioned_numel
293
- total_params += 1
294
-
295
- if debug:
296
- print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
297
- state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
298
- offset += unpartitioned_numel
299
-
300
- # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
301
- # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
302
- # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
303
- # live optimizer object, so we are checking that the numbers are within the right range
304
- align_to = 2 * world_size
305
-
306
- def zero2_align(x):
307
- return align_to * math.ceil(x / align_to)
308
-
309
- if debug:
310
- print(f"original offset={offset}, avail_numel={avail_numel}")
311
-
312
- offset = zero2_align(offset)
313
- avail_numel = zero2_align(avail_numel)
314
-
315
- if debug:
316
- print(f"aligned offset={offset}, avail_numel={avail_numel}")
317
-
318
- # Sanity check
319
- if offset != avail_numel:
320
- raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
321
-
322
- print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
323
-
324
-
325
- def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
326
- exclude_frozen_parameters):
327
- state_dict = OrderedDict()
328
-
329
- # buffers
330
- buffers = zero_model_states[0].buffers
331
- state_dict.update(buffers)
332
- if debug:
333
- print(f"added {len(buffers)} buffers")
334
-
335
- if not exclude_frozen_parameters:
336
- _zero2_merge_frozen_params(state_dict, zero_model_states)
337
-
338
- _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
339
-
340
- # recover shared parameters
341
- for pair in zero_model_states[0].shared_params:
342
- if pair[1] in state_dict:
343
- state_dict[pair[0]] = state_dict[pair[1]]
344
-
345
- return state_dict
346
-
347
-
348
- def zero3_partitioned_param_info(unpartitioned_numel, world_size):
349
- remainder = unpartitioned_numel % world_size
350
- padding_numel = (world_size - remainder) if remainder else 0
351
- partitioned_numel = math.ceil(unpartitioned_numel / world_size)
352
- return partitioned_numel, padding_numel
353
-
354
-
355
- def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
356
- if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
357
- return
358
-
359
- if debug:
360
- for i in range(world_size):
361
- num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
362
- print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
363
-
364
- frozen_param_shapes = zero_model_states[0].frozen_param_shapes
365
- wanted_params = len(frozen_param_shapes)
366
- wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
367
- avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
368
- print(f'Frozen params: Have {avail_numel} numels to process.')
369
- print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
370
-
371
- total_params = 0
372
- total_numel = 0
373
- for name, shape in zero_model_states[0].frozen_param_shapes.items():
374
- total_params += 1
375
- unpartitioned_numel = shape.numel()
376
- total_numel += unpartitioned_numel
377
-
378
- param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
379
- state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
380
-
381
- partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
382
-
383
- if debug:
384
- print(
385
- f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
386
- )
387
-
388
- print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
389
-
390
-
391
- class GatheredTensor:
392
- """
393
- A pseudo tensor that collects partitioned weights.
394
- It is more memory efficient when there are multiple groups.
395
- """
396
-
397
- def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
398
- self.flat_groups = flat_groups
399
- self.flat_groups_offset = flat_groups_offset
400
- self.offset = offset
401
- self.partitioned_numel = partitioned_numel
402
- self.shape = shape
403
- self.dtype = self.flat_groups[0][0].dtype
404
-
405
- def contiguous(self):
406
- """
407
- Merge partitioned weights from flat_groups into a single tensor.
408
- """
409
- end_idx = self.offset + self.partitioned_numel
410
- world_size = len(self.flat_groups)
411
- pad_flat_param_chunks = []
412
-
413
- for rank_i in range(world_size):
414
- # for each rank, we need to collect weights from related group/groups
415
- flat_groups_at_rank_i = self.flat_groups[rank_i]
416
- start_group_id = None
417
- end_group_id = None
418
- for group_id in range(len(self.flat_groups_offset)):
419
- if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
420
- start_group_id = group_id
421
- if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
422
- end_group_id = group_id
423
- break
424
- # collect weights from related group/groups
425
- for group_id in range(start_group_id, end_group_id + 1):
426
- flat_tensor = flat_groups_at_rank_i[group_id]
427
- start_offset = self.offset - self.flat_groups_offset[group_id]
428
- end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
429
- pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
430
-
431
- # collect weights from all ranks
432
- pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
433
- param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
434
- return param
435
-
436
-
437
- def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
438
- param_shapes = zero_model_states[0].param_shapes
439
- avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
440
-
441
- # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
442
- # param, re-consolidating each param, while dealing with padding if any
443
-
444
- # merge list of dicts, preserving order
445
- param_shapes = {k: v for d in param_shapes for k, v in d.items()}
446
-
447
- if debug:
448
- for i in range(world_size):
449
- print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
450
-
451
- wanted_params = len(param_shapes)
452
- wanted_numel = sum(shape.numel() for shape in param_shapes.values())
453
- # not asserting if there is a mismatch due to possible padding
454
- avail_numel = fp32_flat_groups[0].numel() * world_size
455
- print(f"Trainable params: Have {avail_numel} numels to process.")
456
- print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
457
-
458
- # params
459
- # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
460
- # out-of-core computing solution
461
- offset = 0
462
- total_numel = 0
463
- total_params = 0
464
- flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
465
- for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
466
- unpartitioned_numel = shape.numel()
467
- total_numel += unpartitioned_numel
468
- total_params += 1
469
- partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
470
-
471
- if debug:
472
- print(
473
- f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
474
- )
475
-
476
- # memory efficient tensor
477
- tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
478
- state_dict[name] = tensor
479
- offset += partitioned_numel
480
-
481
- offset *= world_size
482
-
483
- # Sanity check
484
- if offset != avail_numel:
485
- raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
486
-
487
- print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
488
-
489
-
490
- def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
491
- exclude_frozen_parameters):
492
- state_dict = OrderedDict()
493
-
494
- # buffers
495
- buffers = zero_model_states[0].buffers
496
- state_dict.update(buffers)
497
- if debug:
498
- print(f"added {len(buffers)} buffers")
499
-
500
- if not exclude_frozen_parameters:
501
- _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
502
-
503
- _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
504
-
505
- # recover shared parameters
506
- for pair in zero_model_states[0].shared_params:
507
- if pair[1] in state_dict:
508
- state_dict[pair[0]] = state_dict[pair[1]]
509
-
510
- return state_dict
511
-
512
-
513
- def to_torch_tensor(state_dict, return_empty_tensor=False):
514
- """
515
- Convert state_dict of GatheredTensor to torch tensor
516
- """
517
- torch_state_dict = {}
518
- converted_tensors = {}
519
- for name, tensor in state_dict.items():
520
- tensor_id = id(tensor)
521
- if tensor_id in converted_tensors: # shared tensors
522
- shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
523
- torch_state_dict[name] = shared_tensor
524
- else:
525
- converted_tensors[tensor_id] = name
526
- if return_empty_tensor:
527
- torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
528
- else:
529
- torch_state_dict[name] = tensor.contiguous()
530
- return torch_state_dict
531
-
532
-
533
- def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
534
- tag=None,
535
- exclude_frozen_parameters=False,
536
- lazy_mode=False):
537
- """
538
- Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
539
- ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
540
- via a model hub.
541
-
542
- Args:
543
- - ``checkpoint_dir``: path to the desired checkpoint folder
544
- - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
545
- - ``exclude_frozen_parameters``: exclude frozen parameters
546
- - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
547
- Convert the pesduo tensor to torch tensor by ``.contiguous()``
548
-
549
- Returns:
550
- - pytorch ``state_dict``
551
-
552
- A typical usage might be ::
553
-
554
- from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
555
- # do the training and checkpoint saving
556
- state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
557
- model = model.cpu() # move to cpu
558
- model.load_state_dict(state_dict)
559
- # submit to model hub or save the model to share with others
560
-
561
- In this example the ``model`` will no longer be usable in the deepspeed context of the same
562
- application. i.e. you will need to re-initialize the deepspeed engine, since
563
- ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
564
-
565
- If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
566
-
567
- Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
568
- You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
569
- the checkpoint. Or you can load state_dict in lazy mode ::
570
-
571
- from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
572
- state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
573
- for name, lazy_tensor in state_dict.item():
574
- tensor = lazy_tensor.contiguous() # to cpu
575
- print(name, tensor)
576
- # del tensor to release memory if it no longer in use
577
- """
578
- if tag is None:
579
- latest_path = os.path.join(checkpoint_dir, 'latest')
580
- if os.path.isfile(latest_path):
581
- with open(latest_path, 'r') as fd:
582
- tag = fd.read().strip()
583
- else:
584
- raise ValueError(f"Unable to find 'latest' file at {latest_path}")
585
-
586
- ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
587
-
588
- if not os.path.isdir(ds_checkpoint_dir):
589
- raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
590
-
591
- state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
592
- if lazy_mode:
593
- return state_dict
594
- else:
595
- return to_torch_tensor(state_dict)
596
-
597
-
598
- def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
599
- output_dir,
600
- max_shard_size="5GB",
601
- safe_serialization=False,
602
- tag=None,
603
- exclude_frozen_parameters=False):
604
- """
605
- Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
606
- loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
607
-
608
- Args:
609
- - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
610
- - ``output_dir``: directory to the pytorch fp32 state_dict output files
611
- - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
612
- - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
613
- - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
614
- - ``exclude_frozen_parameters``: exclude frozen parameters
615
- """
616
-
617
- # Dependency pre-check
618
- if safe_serialization:
619
- try:
620
- from safetensors.torch import save_file
621
- except ImportError:
622
- print('If you want to use `safe_serialization`, please `pip install safetensors`')
623
- raise
624
- if max_shard_size is not None:
625
- try:
626
- from huggingface_hub import split_torch_state_dict_into_shards
627
- except ImportError:
628
- print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
629
- raise
630
-
631
- # Convert zero checkpoint to state_dict
632
- state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
633
- tag,
634
- exclude_frozen_parameters,
635
- lazy_mode=True)
636
-
637
- # Shard the model if it is too big.
638
- weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
639
- if max_shard_size is not None:
640
- filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
641
- # an memory-efficient approach for sharding
642
- empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
643
- state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
644
- filename_pattern=filename_pattern,
645
- max_shard_size=max_shard_size)
646
- else:
647
- from collections import namedtuple
648
- StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
649
- state_dict_split = StateDictSplit(is_sharded=False,
650
- filename_to_tensors={weights_name: list(state_dict.keys())})
651
-
652
- # Save the model by shard
653
- os.makedirs(output_dir, exist_ok=True)
654
- filename_to_tensors = state_dict_split.filename_to_tensors.items()
655
- for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
656
- shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
657
- shard_state_dict = to_torch_tensor(shard_state_dict)
658
- output_path = os.path.join(output_dir, shard_file)
659
- if safe_serialization:
660
- save_file(shard_state_dict, output_path, metadata={"format": "pt"})
661
- else:
662
- torch.save(shard_state_dict, output_path)
663
- # release the memory of current shard
664
- for tensor_name in list(shard_state_dict.keys()):
665
- del state_dict[tensor_name]
666
- del shard_state_dict[tensor_name]
667
- del shard_state_dict
668
- gc.collect()
669
-
670
- # Save index if sharded
671
- if state_dict_split.is_sharded:
672
- index = {
673
- "metadata": state_dict_split.metadata,
674
- "weight_map": state_dict_split.tensor_to_filename,
675
- }
676
- save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
677
- save_index_file = os.path.join(output_dir, save_index_file)
678
- with open(save_index_file, "w", encoding="utf-8") as f:
679
- content = json.dumps(index, indent=2, sort_keys=True) + "\n"
680
- f.write(content)
681
-
682
-
683
- def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
684
- """
685
- 1. Put the provided model to cpu
686
- 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
687
- 3. Load it into the provided model
688
-
689
- Args:
690
- - ``model``: the model object to update
691
- - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
692
- - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
693
-
694
- Returns:
695
- - ``model`: modified model
696
-
697
- Make sure you have plenty of CPU memory available before you call this function. If you don't
698
- have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
699
- conveniently placed for you in the checkpoint folder.
700
-
701
- A typical usage might be ::
702
-
703
- from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
704
- model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
705
- # submit to model hub or save the model to share with others
706
-
707
- Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
708
- of the same application. i.e. you will need to re-initialize the deepspeed engine, since
709
- ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
710
-
711
- """
712
- logger.info("Extracting fp32 weights")
713
- state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
714
-
715
- logger.info("Overwriting model with fp32 weights")
716
- model = model.cpu()
717
- model.load_state_dict(state_dict, strict=False)
718
-
719
- return model
720
-
721
-
722
- if __name__ == "__main__":
723
- parser = argparse.ArgumentParser()
724
- parser.add_argument("checkpoint_dir",
725
- type=str,
726
- help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
727
- parser.add_argument("output_dir",
728
- type=str,
729
- help="directory to the pytorch fp32 state_dict output files"
730
- "(e.g. path/checkpoint-12-output/)")
731
- parser.add_argument(
732
- "--max_shard_size",
733
- type=str,
734
- default="5GB",
735
- help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
736
- "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
737
- "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
738
- "without CPU OOM issues.")
739
- parser.add_argument(
740
- "--safe_serialization",
741
- default=False,
742
- action='store_true',
743
- help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
744
- parser.add_argument("-t",
745
- "--tag",
746
- type=str,
747
- default=None,
748
- help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
749
- parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
750
- parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
751
- args = parser.parse_args()
752
-
753
- debug = args.debug
754
-
755
- convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
756
- args.output_dir,
757
- max_shard_size=args.max_shard_size,
758
- safe_serialization=args.safe_serialization,
759
- tag=args.tag,
760
- exclude_frozen_parameters=args.exclude_frozen_parameters)