guanning commited on
Commit
740999b
·
verified ·
1 Parent(s): c2a2be6

train_118786/frontier_lam0.05_seed42: run config and metrics

Browse files
train_118786/frontier_lam0.05_seed42/config.yaml ADDED
@@ -0,0 +1,362 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ data:
2
+ tokenizer: null
3
+ use_shm: false
4
+ train_files:
5
+ - /data/home/guanning/GRACEFUL-SCALING/runs/retry_frontier/data_full/train.parquet
6
+ val_files: []
7
+ prompt_key: prompt
8
+ reward_fn_key: data_source
9
+ max_prompt_length: 2048
10
+ max_response_length: 4096
11
+ train_batch_size: 32
12
+ val_batch_size: null
13
+ return_raw_input_ids: false
14
+ return_raw_chat: false
15
+ return_full_prompt: false
16
+ shuffle: true
17
+ validation_shuffle: false
18
+ filter_overlong_prompts: true
19
+ filter_overlong_prompts_workers: 1
20
+ truncation: error
21
+ apply_chat_template: true
22
+ image_key: images
23
+ video_key: videos
24
+ trust_remote_code: false
25
+ custom_cls:
26
+ path: null
27
+ name: null
28
+ seed: 42
29
+ dataloader_num_workers: 2
30
+ resume_training_parameters:
31
+ dataloader_local_path: null
32
+ resume_global_steps: null
33
+ wandb_id_to_resume: null
34
+ diagnostics:
35
+ grad_norm_threshold_for_saving_checkpoint: null
36
+ checkpoint_dir: null
37
+ actor_rollout_ref:
38
+ hybrid_engine: true
39
+ model:
40
+ path: /data/home/guanning/GRACEFUL-SCALING/models/Reliability-1.7B-final-idk-ckpt-1500
41
+ use_shm: false
42
+ external_lib: null
43
+ override_config: {}
44
+ attn_implementation: flash_attention_2
45
+ enable_gradient_checkpointing: true
46
+ enable_activation_offload: false
47
+ use_remove_padding: false
48
+ lora_rank: 0
49
+ lora_alpha: 16
50
+ target_modules: all-linear
51
+ use_liger: false
52
+ use_fused_kernels: false
53
+ fused_kernel_options:
54
+ impl_backend: torch
55
+ trust_remote_code: false
56
+ actor:
57
+ strategy: fsdp
58
+ dtype: bfloat16
59
+ ppo_mini_batch_size: 32
60
+ ppo_micro_batch_size: null
61
+ ppo_micro_batch_size_per_gpu: 4
62
+ use_dynamic_bsz: false
63
+ ppo_max_token_len_per_gpu: 16384
64
+ grad_clip: 1.0
65
+ clip_ratio: 0.2
66
+ clip_ratio_low: 0.2
67
+ clip_ratio_high: 0.2
68
+ policy_loss:
69
+ loss_mode: vanilla
70
+ clip_cov_ratio: 0.0002
71
+ clip_cov_lb: 1.0
72
+ clip_cov_ub: 5.0
73
+ kl_cov_ratio: 0.0002
74
+ ppo_kl_coef: 0.1
75
+ clip_ratio_c: 3.0
76
+ loss_agg_mode: seq-mean-token-sum
77
+ entropy_coeff: 0.0
78
+ use_kl_loss: false
79
+ use_torch_compile: true
80
+ kl_loss_coef: 0.001
81
+ kl_loss_type: low_var_kl
82
+ ppo_epochs: 1
83
+ shuffle: false
84
+ ulysses_sequence_parallel_size: 1
85
+ entropy_from_logits_with_chunking: false
86
+ entropy_checkpointing: false
87
+ checkpoint:
88
+ save_contents:
89
+ - model
90
+ - optimizer
91
+ - extra
92
+ - hf_model
93
+ load_contents:
94
+ - model
95
+ - optimizer
96
+ - extra
97
+ optim:
98
+ lr: 1.0e-06
99
+ lr_warmup_steps: -1
100
+ lr_warmup_steps_ratio: 0.0
101
+ min_lr_ratio: 0.0
102
+ num_cycles: 0.5
103
+ warmup_style: constant
104
+ total_training_steps: -1
105
+ weight_decay: 0.0
106
+ fsdp_config:
107
+ wrap_policy:
108
+ min_num_params: 0
109
+ param_offload: false
110
+ optimizer_offload: false
111
+ offload_policy: false
112
+ reshard_after_forward: true
113
+ fsdp_size: -1
114
+ forward_prefetch: false
115
+ profiler:
116
+ discrete: false
117
+ all_ranks: false
118
+ ranks: null
119
+ ref:
120
+ strategy: fsdp
121
+ fsdp_config:
122
+ param_offload: false
123
+ reshard_after_forward: true
124
+ forward_prefetch: false
125
+ wrap_policy:
126
+ min_num_params: 0
127
+ use_torch_compile: true
128
+ log_prob_micro_batch_size: null
129
+ log_prob_micro_batch_size_per_gpu: null
130
+ log_prob_use_dynamic_bsz: false
131
+ log_prob_max_token_len_per_gpu: 16384
132
+ ulysses_sequence_parallel_size: 1
133
+ entropy_from_logits_with_chunking: false
134
+ entropy_checkpointing: false
135
+ profiler:
136
+ discrete: false
137
+ all_ranks: false
138
+ ranks: null
139
+ rollout:
140
+ name: vllm
141
+ mode: sync
142
+ temperature: 1.0
143
+ top_k: -1
144
+ top_p: 1.0
145
+ use_fire_sampling: false
146
+ prompt_length: 2048
147
+ response_length: 4096
148
+ dtype: bfloat16
149
+ gpu_memory_utilization: 0.7
150
+ ignore_eos: false
151
+ enforce_eager: false
152
+ free_cache_engine: false
153
+ load_format: dummy_dtensor
154
+ layered_summon: false
155
+ tensor_model_parallel_size: 1
156
+ max_num_batched_tokens: 8192
157
+ max_model_len: 6144
158
+ max_num_seqs: 1024
159
+ log_prob_micro_batch_size: null
160
+ log_prob_micro_batch_size_per_gpu: 4
161
+ log_prob_use_dynamic_bsz: false
162
+ log_prob_max_token_len_per_gpu: 16384
163
+ disable_log_stats: true
164
+ enable_chunked_prefill: true
165
+ do_sample: true
166
+ 'n': 64
167
+ engine_kwargs:
168
+ vllm:
169
+ swap_space: null
170
+ disable_mm_preprocessor_cache: false
171
+ sglang:
172
+ attention_backend: null
173
+ val_kwargs:
174
+ top_k: -1
175
+ top_p: 1.0
176
+ temperature: 0
177
+ 'n': 1
178
+ do_sample: false
179
+ gen_batch_size: null
180
+ multi_turn:
181
+ enable: false
182
+ max_turns: null
183
+ tool_config_path: null
184
+ completion_callback: null
185
+ use_inference_chat_template: false
186
+ enable_tokenization_sanity_check: true
187
+ calculate_log_probs: false
188
+ profiler:
189
+ discrete: false
190
+ all_ranks: false
191
+ ranks: null
192
+ engine_seed: 42
193
+ critic:
194
+ rollout_n: 64
195
+ strategy: fsdp
196
+ optim:
197
+ lr: 1.0e-05
198
+ lr_warmup_steps_ratio: 0.0
199
+ min_lr_ratio: null
200
+ warmup_style: constant
201
+ total_training_steps: -1
202
+ weight_decay: 0.01
203
+ model:
204
+ path: ~/models/deepseek-llm-7b-chat
205
+ use_shm: false
206
+ tokenizer_path: /data/home/guanning/GRACEFUL-SCALING/models/Reliability-1.7B-final-idk-ckpt-1500
207
+ override_config: {}
208
+ external_lib: null
209
+ enable_gradient_checkpointing: true
210
+ enable_activation_offload: false
211
+ use_remove_padding: false
212
+ trust_remote_code: false
213
+ fsdp_config:
214
+ param_offload: false
215
+ optimizer_offload: false
216
+ offload_policy: false
217
+ reshard_after_forward: true
218
+ wrap_policy:
219
+ min_num_params: 0
220
+ fsdp_size: -1
221
+ forward_prefetch: false
222
+ lora_rank: 0
223
+ lora_alpha: 16
224
+ target_modules: all-linear
225
+ ppo_mini_batch_size: 32
226
+ ppo_micro_batch_size: null
227
+ ppo_micro_batch_size_per_gpu: null
228
+ forward_micro_batch_size: null
229
+ forward_micro_batch_size_per_gpu: null
230
+ use_dynamic_bsz: false
231
+ ppo_max_token_len_per_gpu: 32768
232
+ forward_max_token_len_per_gpu: 32768
233
+ ulysses_sequence_parallel_size: 1
234
+ ppo_epochs: 1
235
+ shuffle: false
236
+ grad_clip: 1.0
237
+ cliprange_value: 0.5
238
+ loss_agg_mode: seq-mean-token-sum
239
+ checkpoint:
240
+ save_contents:
241
+ - model
242
+ - optimizer
243
+ - extra
244
+ load_contents:
245
+ - model
246
+ - optimizer
247
+ - extra
248
+ profiler:
249
+ discrete: false
250
+ all_ranks: false
251
+ ranks: null
252
+ reward_model:
253
+ enable: false
254
+ strategy: fsdp
255
+ model:
256
+ input_tokenizer: /data/home/guanning/GRACEFUL-SCALING/models/Reliability-1.7B-final-idk-ckpt-1500
257
+ path: ~/models/FsfairX-LLaMA3-RM-v0.1
258
+ use_shm: false
259
+ external_lib: null
260
+ use_remove_padding: false
261
+ use_fused_kernels: false
262
+ trust_remote_code: false
263
+ fsdp_config:
264
+ wrap_policy:
265
+ min_num_params: 0
266
+ param_offload: false
267
+ reshard_after_forward: true
268
+ fsdp_size: -1
269
+ forward_prefetch: false
270
+ micro_batch_size: null
271
+ micro_batch_size_per_gpu: null
272
+ max_length: null
273
+ ulysses_sequence_parallel_size: 1
274
+ use_dynamic_bsz: false
275
+ forward_max_token_len_per_gpu: 32768
276
+ reward_manager: multi_thread_idk
277
+ launch_reward_fn_async: false
278
+ sandbox_fusion:
279
+ url: null
280
+ max_concurrent: 64
281
+ profiler:
282
+ discrete: false
283
+ all_ranks: false
284
+ ranks: null
285
+ reward_kwargs:
286
+ num_reward_actors: 32
287
+ custom_reward_function:
288
+ path: null
289
+ name: compute_score
290
+ algorithm:
291
+ gamma: 1.0
292
+ lam: 1.0
293
+ adv_estimator: retry_frontier
294
+ norm_adv_by_std_in_grpo: false
295
+ retry_frontier:
296
+ cost_lambda: 0.05
297
+ tokens_per_unit: 1000.0
298
+ max_samples_per_prompt: 128
299
+ pass_k: 1
300
+ maxrl_smoothed_threshold: 0.125
301
+ advantage_type: grpo
302
+ use_kl_in_reward: false
303
+ truncate_order: 16
304
+ kl_penalty: kl
305
+ kl_ctrl:
306
+ type: fixed
307
+ kl_coef: 0.0
308
+ horizon: 10000
309
+ target_kl: 0.1
310
+ use_pf_ppo: false
311
+ pf_ppo:
312
+ reweight_method: pow
313
+ weight_pow: 2.0
314
+ trainer:
315
+ balance_batch: false
316
+ total_epochs: 10000
317
+ total_training_steps: 10000
318
+ profile_steps: null
319
+ controller_nsight_options:
320
+ trace: cuda,nvtx,cublas,ucx
321
+ cuda-memory-usage: 'true'
322
+ cuda-graph-trace: graph
323
+ worker_nsight_options:
324
+ trace: cuda,nvtx,cublas,ucx
325
+ cuda-memory-usage: 'true'
326
+ cuda-graph-trace: graph
327
+ capture-range: cudaProfilerApi
328
+ capture-range-end: null
329
+ kill: none
330
+ project_name: Reliability-1.7B-retry-frontier
331
+ experiment_name: frontier_lam0.05_seed42
332
+ logger:
333
+ - console
334
+ - wandb
335
+ log_val_generations: 0
336
+ rollout_data_dir: null
337
+ validation_data_dir: null
338
+ nnodes: 1
339
+ n_gpus_per_node: 8
340
+ save_freq: 20
341
+ resume_mode: disable
342
+ resume_from_path: null
343
+ val_before_train: false
344
+ val_only: false
345
+ val_on_last_step: false
346
+ test_freq: -1
347
+ eval_on_last_step: false
348
+ critic_warmup: 0
349
+ default_hdfs_dir: null
350
+ del_local_ckpt_after_load: false
351
+ default_local_dir: /data/home/guanning/GRACEFUL-SCALING/runs/retry_frontier/train_118786/frontier_lam0.05_seed42
352
+ max_actor_ckpt_to_keep: null
353
+ max_critic_ckpt_to_keep: null
354
+ ray_wait_register_center_timeout: 300
355
+ device: cuda
356
+ validation_enabled: false
357
+ hf_history_latest_full: true
358
+ experiment_metrics_path: /data/home/guanning/GRACEFUL-SCALING/runs/retry_frontier/train_118786/frontier_lam0.05_seed42/metrics.jsonl
359
+ ray_init:
360
+ num_cpus: 156
361
+ timeline_json_file: null
362
+ ray_dir: /tmp/retry-ray-243002506-859682
train_118786/frontier_lam0.05_seed42/experiment.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "arm": "frontier",
3
+ "seed": 42,
4
+ "c": 0.4,
5
+ "cost_lambda": 0.05,
6
+ "model_path": "/data/home/guanning/GRACEFUL-SCALING/models/Reliability-1.7B-final-idk-ckpt-1500",
7
+ "model_source": {
8
+ "repo": "daman1209arora/Reliability-1.7B-final-idk-ckpt-1500",
9
+ "revision": "a841f88d1430dde795ebedb08fcd4dff1d9bc8d5"
10
+ },
11
+ "data_manifest": {
12
+ "mode": "full-train",
13
+ "c": 0.4,
14
+ "train_size": 17398,
15
+ "sources": {
16
+ "train": {
17
+ "path": "/data/home/guanning/GRACEFUL-SCALING/failing-gracefully/llm_math/scripts/data/dapo/idk/train_c_0.4.parquet",
18
+ "sha256": "b0687116e6c77bf8df32831c95f977b1c810a921257fe5ecc07d0c9141fe0ac5"
19
+ }
20
+ },
21
+ "files": {
22
+ "train.parquet": "b0687116e6c77bf8df32831c95f977b1c810a921257fe5ecc07d0c9141fe0ac5"
23
+ },
24
+ "training_note": "Exact original parquet copy: no sampling, deduplication, relabeling or evaluation-overlap filtering."
25
+ },
26
+ "git_commit": "eec18425f1d1b4fca65d488fe7339fa05c12b6b7",
27
+ "git_status": " M .gitignore\n M README.md\n M experiments/retry_frontier/README.md\n M experiments/retry_frontier/train.py\n M failing-gracefully/llm_math/verl/protocol.py\n M failing-gracefully/llm_math/verl/trainer/ppo/ray_trainer.py\n M failing-gracefully/llm_math/verl/trainer/ppo/retry_frontier.py\n M failing-gracefully/llm_math/verl/utils/tracking.py\n M kempner/retry_frontier.sbatch\n M tests/test_retry_frontier.py\n?? coreweave/\n?? failing-gracefully/llm_math/verl/models/\n",
28
+ "dry_run": false,
29
+ "config_sha256": "d6a22ebd94523f81989577be0416c7fa2190a0b2c6a8a40f365caf2164bb812c",
30
+ "objective": "infinite_retry_q_minus_lambda_C",
31
+ "notes": "Fixed-c matched-start proof of concept; PPO and finite-sample/adaptive statistics are approximate. Historical HF exports and latest full training state retained; no online validation."
32
+ }
train_118786/frontier_lam0.05_seed42/metrics.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
train_118786/frontier_lam0.05_seed42/wandb_id.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ sp20z9cp