Upload 11 files

.gitattributes

@@ -0,0 +1,48 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*.tfevents* filter=lfs diff=lfs merge=lfs -text
+*.db* filter=lfs diff=lfs merge=lfs -text
+*.ark* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.gguf* filter=lfs diff=lfs merge=lfs -text
+*.ggml filter=lfs diff=lfs merge=lfs -text
+*.llamafile* filter=lfs diff=lfs merge=lfs -text
+*.pt2 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+"/tokenizer.json" filter=lfs diff=lfs merge=lfs -text

chat_template.jinja

@@ -0,0 +1,26 @@
+{% if not add_generation_prompt is defined %}
+{% set add_generation_prompt = false %}
+{% endif %}
+{%- set ns = namespace(found=false) -%}
+{%- for message in messages -%}
+    {%- if message['role'] == 'system' -%}
+        {%- set ns.found = true -%}
+    {%- endif -%}
+{%- endfor -%}
+{{bos_token}}{%- if not ns.found -%}
+{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n'}}
+{%- endif %}
+{%- for message in messages %}
+    {%- if message['role'] == 'system' %}
+{{ message['content'] }}
+    {%- else %}
+        {%- if message['role'] == 'user' %}
+{{'### Instruction:\n' + message['content'] + '\n'}}
+        {%- else %}
+{{'### Response:\n' + message['content'] + '\n<|EOT|>\n'}}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{% if add_generation_prompt %}
+{{'### Response:'}}
+{% endif %}

configuration.json

@@ -0,0 +1 @@
+{"task":"other"}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:045b8a08ca86336e5664357c58e7c6ade1e5364e2cefbe1afb98f363d80b1695
+oid sha256:3eff1992b978ce82621b65a00d38645faf2fae0309abffc7b189fa84ddc7757e
 size 2692969128

tokenizer_config.json

@@ -195,11 +195,10 @@
   "legacy": true,
   "model_max_length": 16384,
   "pad_token": "<｜end▁of▁sentence｜>",
-  "padding_side": "left",
+  "padding_side": "right",
   "sp_model_kwargs": {},
   "split_special_tokens": false,
   "tokenizer_class": "LlamaTokenizerFast",
-  "truncation_side": "left",
   "unk_token": null,
   "use_default_system_prompt": false
 }

trainer_state.json

@@ -2,147 +2,1319 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.6153846153846154,
+  "epoch": 1.6025641025641026,
   "eval_steps": 500,
-  "global_step": 48,
+  "global_step": 500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0943359375,
-      "completions/max_length": 4096.0,
-      "completions/max_terminated_length": 3681.1,
-      "completions/mean_length": 1021.4728515625,
-      "completions/mean_terminated_length": 701.1630737304688,
-      "completions/min_length": 103.7,
-      "completions/min_terminated_length": 103.7,
-      "entropy": 0.17240249924361706,
-      "epoch": 0.1282051282051282,
-      "frac_reward_zero_std": 0.0515625,
-      "grad_norm": 0.00012006821180025731,
-      "learning_rate": 5.625e-06,
-      "loss": -0.0001,
-      "num_tokens": 10714829.0,
-      "reward": -0.35699094235897066,
-      "reward_std": 0.3353965669870377,
-      "rewards/grpo_reward_function/mean": -0.35699094235897066,
-      "rewards/grpo_reward_function/std": 0.7091993272304535,
-      "sampling/importance_sampling_ratio/max": 0.1970108050853014,
-      "sampling/importance_sampling_ratio/mean": 0.0012754969735397025,
-      "sampling/importance_sampling_ratio/min": 0.0,
-      "sampling/sampling_logp_difference/max": 1.831837797164917,
-      "sampling/sampling_logp_difference/mean": 0.0368979848921299,
+      "clip_ratio/high_max": 0.0031111116637475788,
+      "clip_ratio/high_mean": 0.0016541311924811453,
+      "clip_ratio/low_mean": 0.0013736484514083713,
+      "clip_ratio/low_min": 0.000607477161247516,
+      "clip_ratio/region_mean": 0.003027779620606452,
+      "completion_length": 4608.88515625,
+      "completions/clipped_ratio": 0.678125,
+      "completions/max_length": 14070.2,
+      "completions/max_terminated_length": 11684.8,
+      "completions/mean_length": 4608.88515625,
+      "completions/mean_terminated_length": 3749.20771484375,
+      "completions/min_length": 303.0,
+      "completions/min_terminated_length": 303.0,
+      "epoch": 0.03205128205128205,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.037186067551374435,
+      "kl": 0.0009099063114263118,
+      "learning_rate": 4.5e-07,
+      "loss": 0.0004,
+      "num_tokens": 7255001.0,
+      "reward": -1.5566178560256958,
+      "reward_std": 0.9402843713760376,
       "step": 10,
-      "step_time": 179.07797421738505
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.078125,
-      "completions/max_length": 4096.0,
-      "completions/max_terminated_length": 3652.6,
-      "completions/mean_length": 961.977734375,
-      "completions/mean_terminated_length": 696.3083312988281,
-      "completions/min_length": 86.3,
-      "completions/min_terminated_length": 86.3,
-      "entropy": 0.17260038182139398,
-      "epoch": 0.2564102564102564,
-      "frac_reward_zero_std": 0.04453125,
-      "grad_norm": 0.00011115181306767518,
-      "learning_rate": 9.942341621640558e-06,
-      "loss": -0.0003,
-      "num_tokens": 21190031.0,
-      "reward": -0.3344299763441086,
-      "reward_std": 0.32805776298046113,
-      "rewards/grpo_reward_function/mean": -0.3344299763441086,
-      "rewards/grpo_reward_function/std": 0.7096675753593444,
-      "sampling/importance_sampling_ratio/max": 0.21866981824859977,
-      "sampling/importance_sampling_ratio/mean": 0.0013687322207260877,
-      "sampling/importance_sampling_ratio/min": 0.0,
-      "sampling/sampling_logp_difference/max": 1.826985728740692,
-      "sampling/sampling_logp_difference/mean": 0.03691292852163315,
+      "step_time": 293.0822082865983
+    },
+    {
+      "clip_ratio/high_max": 0.0039304225007072095,
+      "clip_ratio/high_mean": 0.0022670914826449008,
+      "clip_ratio/low_mean": 0.0017602394160348921,
+      "clip_ratio/low_min": 0.0006693943403661251,
+      "clip_ratio/region_mean": 0.0040273308870382605,
+      "completion_length": 4351.56328125,
+      "completions/clipped_ratio": 0.67734375,
+      "completions/max_length": 13687.0,
+      "completions/max_terminated_length": 11181.2,
+      "completions/mean_length": 4351.56328125,
+      "completions/mean_terminated_length": 3482.92275390625,
+      "completions/min_length": 464.8,
+      "completions/min_terminated_length": 548.0,
+      "epoch": 0.0641025641025641,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.034340787678956985,
+      "kl": 0.001333014338160865,
+      "learning_rate": 9.499999999999999e-07,
+      "loss": 0.0005,
+      "num_tokens": 14183526.0,
+      "reward": -1.5493804216384888,
+      "reward_std": 0.8581619977951049,
       "step": 20,
-      "step_time": 178.9407274030149
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0720703125,
-      "completions/max_length": 4096.0,
-      "completions/max_terminated_length": 3748.0,
-      "completions/mean_length": 916.4140625,
-      "completions/mean_terminated_length": 670.6310546875,
-      "completions/min_length": 91.7,
-      "completions/min_terminated_length": 91.7,
-      "entropy": 0.17158033028244973,
-      "epoch": 0.38461538461538464,
-      "frac_reward_zero_std": 0.046875,
-      "grad_norm": 0.00029510410537975974,
-      "learning_rate": 8.953878684688492e-06,
-      "loss": -0.0,
-      "num_tokens": 31354631.0,
-      "reward": -0.3493777960538864,
-      "reward_std": 0.3407163262367249,
-      "rewards/grpo_reward_function/mean": -0.34937779903411864,
-      "rewards/grpo_reward_function/std": 0.7120605766773224,
-      "sampling/importance_sampling_ratio/max": 0.3080031927675009,
-      "sampling/importance_sampling_ratio/mean": 0.002376599394483492,
-      "sampling/importance_sampling_ratio/min": 0.0,
-      "sampling/sampling_logp_difference/max": 2.0400707840919496,
-      "sampling/sampling_logp_difference/mean": 0.03684176206588745,
+      "step_time": 284.4924844510853
+    },
+    {
+      "clip_ratio/high_max": 0.003806740918662399,
+      "clip_ratio/high_mean": 0.002326158672804013,
+      "clip_ratio/low_mean": 0.001812949904706329,
+      "clip_ratio/low_min": 0.0006994255629251712,
+      "clip_ratio/region_mean": 0.004139108571689576,
+      "completion_length": 4433.5515625,
+      "completions/clipped_ratio": 0.69921875,
+      "completions/max_length": 13895.4,
+      "completions/max_terminated_length": 11343.8,
+      "completions/mean_length": 4433.5515625,
+      "completions/mean_terminated_length": 3893.72841796875,
+      "completions/min_length": 318.2,
+      "completions/min_terminated_length": 327.8,
+      "epoch": 0.09615384615384616,
+      "frac_reward_zero_std": 0.071875,
+      "grad_norm": 0.04159880056977272,
+      "kl": 0.0048721997620305045,
+      "learning_rate": 1e-06,
+      "loss": 0.0005,
+      "num_tokens": 21250332.0,
+      "reward": -1.4585292100906373,
+      "reward_std": 0.9341987490653991,
       "step": 30,
-      "step_time": 171.81590754203498
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0662109375,
-      "completions/max_length": 4096.0,
-      "completions/max_terminated_length": 3509.1,
-      "completions/mean_length": 912.234375,
-      "completions/mean_terminated_length": 686.69970703125,
-      "completions/min_length": 83.6,
-      "completions/min_terminated_length": 83.6,
-      "entropy": 0.16810744479298592,
-      "epoch": 0.5128205128205128,
-      "frac_reward_zero_std": 0.06328125,
-      "grad_norm": 1.6541331856452166e-05,
-      "learning_rate": 6.971779275566593e-06,
-      "loss": -0.0001,
-      "num_tokens": 41526511.0,
-      "reward": -0.3015082895755768,
-      "reward_std": 0.3079641282558441,
-      "rewards/grpo_reward_function/mean": -0.301508292555809,
-      "rewards/grpo_reward_function/std": 0.729301530122757,
-      "sampling/importance_sampling_ratio/max": 0.3999401656910777,
-      "sampling/importance_sampling_ratio/mean": 0.003650264849420637,
-      "sampling/importance_sampling_ratio/min": 1.04037442665946e-40,
-      "sampling/sampling_logp_difference/max": 1.7756866335868835,
-      "sampling/sampling_logp_difference/mean": 0.0364519115537405,
+      "step_time": 291.55923374779525
+    },
+    {
+      "clip_ratio/high_max": 0.004458311991766095,
+      "clip_ratio/high_mean": 0.002367599733406678,
+      "clip_ratio/low_mean": 0.0017874852346722037,
+      "clip_ratio/low_min": 0.0006575059236638481,
+      "clip_ratio/region_mean": 0.004155084933154285,
+      "completion_length": 4515.86640625,
+      "completions/clipped_ratio": 0.7109375,
+      "completions/max_length": 13925.0,
+      "completions/max_terminated_length": 11563.0,
+      "completions/mean_length": 4515.86640625,
+      "completions/mean_terminated_length": 3729.900390625,
+      "completions/min_length": 370.2,
+      "completions/min_terminated_length": 382.6,
+      "epoch": 0.1282051282051282,
+      "frac_reward_zero_std": 0.046875,
+      "grad_norm": 0.03651541844010353,
+      "kl": 0.013490893482230603,
+      "learning_rate": 1e-06,
+      "loss": 0.0008,
+      "num_tokens": 28395169.0,
+      "reward": -1.3941103219985962,
+      "reward_std": 0.965558671951294,
       "step": 40,
-      "step_time": 178.55749775655568
+      "step_time": 288.6464111253619
+    },
+    {
+      "clip_ratio/high_max": 0.0038024169276468455,
+      "clip_ratio/high_mean": 0.0022123225033283235,
+      "clip_ratio/low_mean": 0.0014963536988943816,
+      "clip_ratio/low_min": 0.0006316808693100029,
+      "clip_ratio/region_mean": 0.0037086761905811725,
+      "completion_length": 4612.26015625,
+      "completions/clipped_ratio": 0.70546875,
+      "completions/max_length": 14283.2,
+      "completions/max_terminated_length": 11699.4,
+      "completions/mean_length": 4612.26015625,
+      "completions/mean_terminated_length": 3731.177685546875,
+      "completions/min_length": 198.0,
+      "completions/min_terminated_length": 198.0,
+      "epoch": 0.16025641025641027,
+      "frac_reward_zero_std": 0.06875,
+      "grad_norm": 0.03204132243990898,
+      "kl": 0.014296455890871584,
+      "learning_rate": 1e-06,
+      "loss": 0.0008,
+      "num_tokens": 35643482.0,
+      "reward": -1.1730658531188964,
+      "reward_std": 0.9515015006065368,
+      "step": 50,
+      "step_time": 292.537625066936
+    },
+    {
+      "clip_ratio/high_max": 0.003942539077252149,
+      "clip_ratio/high_mean": 0.002266978449188173,
+      "clip_ratio/low_mean": 0.001514440990285948,
+      "clip_ratio/low_min": 0.0006650009563600179,
+      "clip_ratio/region_mean": 0.0037814194802194834,
+      "completion_length": 4702.503125,
+      "completions/clipped_ratio": 0.69765625,
+      "completions/max_length": 14117.8,
+      "completions/max_terminated_length": 11636.4,
+      "completions/mean_length": 4702.503125,
+      "completions/mean_terminated_length": 3853.881689453125,
+      "completions/min_length": 352.0,
+      "completions/min_terminated_length": 352.0,
+      "epoch": 0.19230769230769232,
+      "frac_reward_zero_std": 0.05625,
+      "grad_norm": 0.027020364999771118,
+      "kl": 0.020487962337210775,
+      "learning_rate": 1e-06,
+      "loss": 0.0009,
+      "num_tokens": 43053098.0,
+      "reward": -1.1392094850540162,
+      "reward_std": 0.9998014450073243,
+      "step": 60,
+      "step_time": 291.96177350580695
+    },
+    {
+      "clip_ratio/high_max": 0.003909006132744253,
+      "clip_ratio/high_mean": 0.0023349625116679817,
+      "clip_ratio/low_mean": 0.0014747101289685816,
+      "clip_ratio/low_min": 0.0006804624077631161,
+      "clip_ratio/region_mean": 0.0038096726522780953,
+      "completion_length": 4625.7171875,
+      "completions/clipped_ratio": 0.73671875,
+      "completions/max_length": 13927.8,
+      "completions/max_terminated_length": 11652.6,
+      "completions/mean_length": 4625.7171875,
+      "completions/mean_terminated_length": 3635.2849609375,
+      "completions/min_length": 300.6,
+      "completions/min_terminated_length": 300.6,
+      "epoch": 0.22435897435897437,
+      "frac_reward_zero_std": 0.059375,
+      "grad_norm": 0.03386739641427994,
+      "kl": 0.028998210467398165,
+      "learning_rate": 1e-06,
+      "loss": 0.0011,
+      "num_tokens": 50294504.0,
+      "reward": -1.0887041330337524,
+      "reward_std": 0.9451036572456359,
+      "step": 70,
+      "step_time": 289.63046541959045
+    },
+    {
+      "clip_ratio/high_max": 0.0034828576259315013,
+      "clip_ratio/high_mean": 0.002133827318903059,
+      "clip_ratio/low_mean": 0.0013159020396415143,
+      "clip_ratio/low_min": 0.0005153648708073888,
+      "clip_ratio/region_mean": 0.003449729410931468,
+      "completion_length": 4955.690625,
+      "completions/clipped_ratio": 0.75390625,
+      "completions/max_length": 13623.0,
+      "completions/max_terminated_length": 11770.0,
+      "completions/mean_length": 4955.690625,
+      "completions/mean_terminated_length": 4158.72783203125,
+      "completions/min_length": 308.6,
+      "completions/min_terminated_length": 308.6,
+      "epoch": 0.2564102564102564,
+      "frac_reward_zero_std": 0.046875,
+      "grad_norm": 0.02705618552863598,
+      "kl": 0.033334921067580583,
+      "learning_rate": 1e-06,
+      "loss": 0.0013,
+      "num_tokens": 57994324.0,
+      "reward": -0.8986854314804077,
+      "reward_std": 0.9169693470001221,
+      "step": 80,
+      "step_time": 285.659275534004
+    },
+    {
+      "clip_ratio/high_max": 0.0038596913102082906,
+      "clip_ratio/high_mean": 0.0022436348313931375,
+      "clip_ratio/low_mean": 0.0012239705247338861,
+      "clip_ratio/low_min": 0.00046775827640885834,
+      "clip_ratio/region_mean": 0.0034676053561270236,
+      "completion_length": 4770.06484375,
+      "completions/clipped_ratio": 0.73515625,
+      "completions/max_length": 13541.0,
+      "completions/max_terminated_length": 11339.2,
+      "completions/mean_length": 4770.06484375,
+      "completions/mean_terminated_length": 3864.394921875,
+      "completions/min_length": 252.8,
+      "completions/min_terminated_length": 252.8,
+      "epoch": 0.28846153846153844,
+      "frac_reward_zero_std": 0.04375,
+      "grad_norm": 0.02707553654909134,
+      "kl": 0.04342595711350441,
+      "learning_rate": 1e-06,
+      "loss": 0.0016,
+      "num_tokens": 65425615.0,
+      "reward": -0.7649253964424133,
+      "reward_std": 0.8921859622001648,
+      "step": 90,
+      "step_time": 310.5098887000233
+    },
+    {
+      "clip_ratio/high_max": 0.0033430344541557134,
+      "clip_ratio/high_mean": 0.0019853224395774303,
+      "clip_ratio/low_mean": 0.0012781442550476641,
+      "clip_ratio/low_min": 0.0005141039003319747,
+      "clip_ratio/region_mean": 0.0032634666771627963,
+      "completion_length": 4745.64453125,
+      "completions/clipped_ratio": 0.68046875,
+      "completions/max_length": 14155.0,
+      "completions/max_terminated_length": 11576.4,
+      "completions/mean_length": 4745.64453125,
+      "completions/mean_terminated_length": 4113.03388671875,
+      "completions/min_length": 320.4,
+      "completions/min_terminated_length": 320.4,
+      "epoch": 0.32051282051282054,
+      "frac_reward_zero_std": 0.040625,
+      "grad_norm": 0.03889278694987297,
+      "kl": 0.04826167421415448,
+      "learning_rate": 1e-06,
+      "loss": 0.0017,
+      "num_tokens": 72858700.0,
+      "reward": -0.7156296968460083,
+      "reward_std": 0.8925606846809387,
+      "step": 100,
+      "step_time": 289.0918200459331
+    },
+    {
+      "clip_ratio/high_max": 0.0035657792002893986,
+      "clip_ratio/high_mean": 0.0020295545691624284,
+      "clip_ratio/low_mean": 0.001214369456283748,
+      "clip_ratio/low_min": 0.0004860336652200203,
+      "clip_ratio/region_mean": 0.0032439240254461764,
+      "completion_length": 4967.11640625,
+      "completions/clipped_ratio": 0.63046875,
+      "completions/max_length": 13689.2,
+      "completions/max_terminated_length": 12475.8,
+      "completions/mean_length": 4967.11640625,
+      "completions/mean_terminated_length": 4782.764453125,
+      "completions/min_length": 270.8,
+      "completions/min_terminated_length": 270.8,
+      "epoch": 0.3525641025641026,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.026075374335050583,
+      "kl": 0.05202609514817595,
+      "learning_rate": 1e-06,
+      "loss": 0.0017,
+      "num_tokens": 80587957.0,
+      "reward": -0.49178884625434877,
+      "reward_std": 0.9207884073257446,
+      "step": 110,
+      "step_time": 288.6188355412334
+    },
+    {
+      "clip_ratio/high_max": 0.003019184013828635,
+      "clip_ratio/high_mean": 0.001815609377808869,
+      "clip_ratio/low_mean": 0.0011770906596211717,
+      "clip_ratio/low_min": 0.00045191738572611937,
+      "clip_ratio/region_mean": 0.0029927000403404238,
+      "completion_length": 4947.5984375,
+      "completions/clipped_ratio": 0.56640625,
+      "completions/max_length": 13888.0,
+      "completions/max_terminated_length": 12296.8,
+      "completions/mean_length": 4947.5984375,
+      "completions/mean_terminated_length": 4629.6720703125,
+      "completions/min_length": 331.2,
+      "completions/min_terminated_length": 331.2,
+      "epoch": 0.38461538461538464,
+      "frac_reward_zero_std": 0.034375,
+      "grad_norm": 0.025916147977113724,
+      "kl": 0.05580858588218689,
+      "learning_rate": 1e-06,
+      "loss": 0.0018,
+      "num_tokens": 88323367.0,
+      "reward": -0.3498576134443283,
+      "reward_std": 0.8661771297454834,
+      "step": 120,
+      "step_time": 324.00552004165945
+    },
+    {
+      "clip_ratio/high_max": 0.0033033261890523134,
+      "clip_ratio/high_mean": 0.001963199768215418,
+      "clip_ratio/low_mean": 0.001268382158013992,
+      "clip_ratio/low_min": 0.0004899931143881985,
+      "clip_ratio/region_mean": 0.003231581998988986,
+      "completion_length": 4826.7859375,
+      "completions/clipped_ratio": 0.4828125,
+      "completions/max_length": 13704.8,
+      "completions/max_terminated_length": 12385.0,
+      "completions/mean_length": 4826.7859375,
+      "completions/mean_terminated_length": 4860.9404296875,
+      "completions/min_length": 334.8,
+      "completions/min_terminated_length": 334.8,
+      "epoch": 0.4166666666666667,
+      "frac_reward_zero_std": 0.01875,
+      "grad_norm": 0.02699657529592514,
+      "kl": 0.07007892271503806,
+      "learning_rate": 1e-06,
+      "loss": 0.0021,
+      "num_tokens": 95898245.0,
+      "reward": -0.10826636925339699,
+      "reward_std": 1.024531078338623,
+      "step": 130,
+      "step_time": 327.7374393735081
+    },
+    {
+      "clip_ratio/high_max": 0.0030151082435622813,
+      "clip_ratio/high_mean": 0.002003528241766617,
+      "clip_ratio/low_mean": 0.0010902579320827498,
+      "clip_ratio/low_min": 0.00040278768283315004,
+      "clip_ratio/region_mean": 0.0030937861418351533,
+      "completion_length": 4827.97734375,
+      "completions/clipped_ratio": 0.44296875,
+      "completions/max_length": 13106.2,
+      "completions/max_terminated_length": 11806.6,
+      "completions/mean_length": 4827.97734375,
+      "completions/mean_terminated_length": 4910.24130859375,
+      "completions/min_length": 390.4,
+      "completions/min_terminated_length": 390.4,
+      "epoch": 0.44871794871794873,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.026754355058073997,
+      "kl": 0.06919310903176665,
+      "learning_rate": 1e-06,
+      "loss": 0.0021,
+      "num_tokens": 103460144.0,
+      "reward": 0.17521797642111778,
+      "reward_std": 0.9395113348960876,
+      "step": 140,
+      "step_time": 287.31414929926393
+    },
+    {
+      "clip_ratio/high_max": 0.003008772386237979,
+      "clip_ratio/high_mean": 0.0019270548655185848,
+      "clip_ratio/low_mean": 0.0009816488542128353,
+      "clip_ratio/low_min": 0.00034070989513566017,
+      "clip_ratio/region_mean": 0.002908703777939081,
+      "completion_length": 4825.0640625,
+      "completions/clipped_ratio": 0.478125,
+      "completions/max_length": 12821.8,
+      "completions/max_terminated_length": 12311.0,
+      "completions/mean_length": 4825.0640625,
+      "completions/mean_terminated_length": 4862.5083984375,
+      "completions/min_length": 289.0,
+      "completions/min_terminated_length": 289.0,
+      "epoch": 0.4807692307692308,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.02678792178630829,
+      "kl": 0.08144378829747438,
+      "learning_rate": 1e-06,
+      "loss": 0.0025,
+      "num_tokens": 111095230.0,
+      "reward": 0.3487085044384003,
+      "reward_std": 0.8401093125343323,
+      "step": 150,
+      "step_time": 287.72478193119167
+    },
+    {
+      "clip_ratio/high_max": 0.002876114356331527,
+      "clip_ratio/high_mean": 0.0018099227454513311,
+      "clip_ratio/low_mean": 0.001024940429488197,
+      "clip_ratio/low_min": 0.00039236449065356284,
+      "clip_ratio/region_mean": 0.00283486315747723,
+      "completion_length": 4726.771875,
+      "completions/clipped_ratio": 0.5234375,
+      "completions/max_length": 13330.4,
+      "completions/max_terminated_length": 12002.0,
+      "completions/mean_length": 4726.771875,
+      "completions/mean_terminated_length": 5123.56083984375,
+      "completions/min_length": 405.2,
+      "completions/min_terminated_length": 405.2,
+      "epoch": 0.5128205128205128,
+      "frac_reward_zero_std": 0.028125,
+      "grad_norm": 0.030215220525860786,
+      "kl": 0.08628348242491483,
+      "learning_rate": 1e-06,
+      "loss": 0.0026,
+      "num_tokens": 118495554.0,
+      "reward": 0.5076338112354278,
+      "reward_std": 0.7434159040451049,
+      "step": 160,
+      "step_time": 323.35687116347253
+    },
+    {
+      "clip_ratio/high_max": 0.0027208653977140785,
+      "clip_ratio/high_mean": 0.0017139136034529656,
+      "clip_ratio/low_mean": 0.0009706232667667791,
+      "clip_ratio/low_min": 0.0003251646208809689,
+      "clip_ratio/region_mean": 0.0026845368323847652,
+      "completion_length": 4803.2765625,
+      "completions/clipped_ratio": 0.51484375,
+      "completions/max_length": 12718.2,
+      "completions/max_terminated_length": 12026.4,
+      "completions/mean_length": 4803.2765625,
+      "completions/mean_terminated_length": 5057.62158203125,
+      "completions/min_length": 365.6,
+      "completions/min_terminated_length": 365.6,
+      "epoch": 0.5448717948717948,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.030473513528704643,
+      "kl": 0.08288778197020293,
+      "learning_rate": 1e-06,
+      "loss": 0.0024,
+      "num_tokens": 126058216.0,
+      "reward": 0.5487113773822785,
+      "reward_std": 0.6891388297080994,
+      "step": 170,
+      "step_time": 291.2210878789425
+    },
+    {
+      "clip_ratio/high_max": 0.0028713531908579172,
+      "clip_ratio/high_mean": 0.0018186487897764892,
+      "clip_ratio/low_mean": 0.0010940565960481762,
+      "clip_ratio/low_min": 0.00038098907243693246,
+      "clip_ratio/region_mean": 0.0029127053683623673,
+      "completion_length": 4646.18984375,
+      "completions/clipped_ratio": 0.54140625,
+      "completions/max_length": 13053.0,
+      "completions/max_terminated_length": 12492.0,
+      "completions/mean_length": 4646.18984375,
+      "completions/mean_terminated_length": 5016.05537109375,
+      "completions/min_length": 419.0,
+      "completions/min_terminated_length": 419.0,
+      "epoch": 0.5769230769230769,
+      "frac_reward_zero_std": 0.01875,
+      "grad_norm": 0.02158021368086338,
+      "kl": 0.08690252397209405,
+      "learning_rate": 1e-06,
+      "loss": 0.0026,
+      "num_tokens": 133370319.0,
+      "reward": 0.4963641226291656,
+      "reward_std": 0.7261568784713746,
+      "step": 180,
+      "step_time": 303.5762791711837
+    },
+    {
+      "clip_ratio/high_max": 0.0032218144508078693,
+      "clip_ratio/high_mean": 0.0018449691764544696,
+      "clip_ratio/low_mean": 0.0011400882038287818,
+      "clip_ratio/low_min": 0.0003823818296950776,
+      "clip_ratio/region_mean": 0.0029850573744624855,
+      "completion_length": 4893.12578125,
+      "completions/clipped_ratio": 0.54765625,
+      "completions/max_length": 12365.2,
+      "completions/max_terminated_length": 11941.2,
+      "completions/mean_length": 4893.12578125,
+      "completions/mean_terminated_length": 5209.0181640625,
+      "completions/min_length": 256.4,
+      "completions/min_terminated_length": 256.4,
+      "epoch": 0.6089743589743589,
+      "frac_reward_zero_std": 0.015625,
+      "grad_norm": 0.026864953339099884,
+      "kl": 0.08798606730997563,
+      "learning_rate": 1e-06,
+      "loss": 0.0026,
+      "num_tokens": 141024392.0,
+      "reward": 0.5374708473682404,
+      "reward_std": 0.7236877918243408,
+      "step": 190,
+      "step_time": 290.9414701361209
+    },
+    {
+      "clip_ratio/high_max": 0.0031956998456735162,
+      "clip_ratio/high_mean": 0.0020226626133080573,
+      "clip_ratio/low_mean": 0.0013265485991723835,
+      "clip_ratio/low_min": 0.0004516207809501793,
+      "clip_ratio/region_mean": 0.0033492111833766103,
+      "completion_length": 4834.32265625,
+      "completions/clipped_ratio": 0.53671875,
+      "completions/max_length": 12988.4,
+      "completions/max_terminated_length": 12323.8,
+      "completions/mean_length": 4834.32265625,
+      "completions/mean_terminated_length": 5249.1677734375,
+      "completions/min_length": 411.8,
+      "completions/min_terminated_length": 411.8,
+      "epoch": 0.6410256410256411,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.03109630011022091,
+      "kl": 0.09753521084785462,
+      "learning_rate": 1e-06,
+      "loss": 0.0028,
+      "num_tokens": 148599113.0,
+      "reward": 0.5115065455436707,
+      "reward_std": 0.6787641048431396,
+      "step": 200,
+      "step_time": 297.2214107103646
+    },
+    {
+      "clip_ratio/high_max": 0.0031499625998549162,
+      "clip_ratio/high_mean": 0.002030467928852886,
+      "clip_ratio/low_mean": 0.0016079474589787424,
+      "clip_ratio/low_min": 0.0004935351942549459,
+      "clip_ratio/region_mean": 0.003638415364548564,
+      "completion_length": 4856.76953125,
+      "completions/clipped_ratio": 0.5734375,
+      "completions/max_length": 12321.6,
+      "completions/max_terminated_length": 12046.8,
+      "completions/mean_length": 4856.76953125,
+      "completions/mean_terminated_length": 5174.89697265625,
+      "completions/min_length": 394.6,
+      "completions/min_terminated_length": 394.6,
+      "epoch": 0.6730769230769231,
+      "frac_reward_zero_std": 0.003125,
+      "grad_norm": 0.03039536438882351,
+      "kl": 0.09620001427829265,
+      "learning_rate": 1e-06,
+      "loss": 0.0029,
+      "num_tokens": 156171442.0,
+      "reward": 0.6326489090919495,
+      "reward_std": 0.5994736194610596,
+      "step": 210,
+      "step_time": 312.2152933940291
+    },
+    {
+      "clip_ratio/high_max": 0.005574445100501179,
+      "clip_ratio/high_mean": 0.0032065427978523074,
+      "clip_ratio/low_mean": 0.006397946062497795,
+      "clip_ratio/low_min": 0.0009641061638831161,
+      "clip_ratio/region_mean": 0.009604489104822278,
+      "completion_length": 5494.121875,
+      "completions/clipped_ratio": 0.5,
+      "completions/max_length": 14614.4,
+      "completions/max_terminated_length": 13020.2,
+      "completions/mean_length": 5494.121875,
+      "completions/mean_terminated_length": 5660.95361328125,
+      "completions/min_length": 452.8,
+      "completions/min_terminated_length": 452.8,
+      "epoch": 0.7051282051282052,
+      "frac_reward_zero_std": 0.003125,
+      "grad_norm": 0.23143450915813446,
+      "kl": 0.13891485184431077,
+      "learning_rate": 1e-06,
+      "loss": 0.0026,
+      "num_tokens": 164621174.0,
+      "reward": 0.2295442931354046,
+      "reward_std": 0.9322809219360352,
+      "step": 220,
+      "step_time": 337.9065022550523
+    },
+    {
+      "clip_ratio/high_max": 0.0029356090584769846,
+      "clip_ratio/high_mean": 0.0016740983992349356,
+      "clip_ratio/low_mean": 0.028463998343795538,
+      "clip_ratio/low_min": 0.013384638354182243,
+      "clip_ratio/region_mean": 0.030138096399605276,
+      "completion_length": 8720.70546875,
+      "completions/clipped_ratio": 0.846875,
+      "completions/max_length": 15289.6,
+      "completions/max_terminated_length": 11706.8,
+      "completions/mean_length": 8720.70546875,
+      "completions/mean_terminated_length": 5305.250537109375,
+      "completions/min_length": 366.0,
+      "completions/min_terminated_length": 467.4,
+      "epoch": 0.7371794871794872,
+      "frac_reward_zero_std": 0.00625,
+      "grad_norm": 0.1857636421918869,
+      "kl": 0.7858731098473072,
+      "learning_rate": 1e-06,
+      "loss": 0.0096,
+      "num_tokens": 177198173.0,
+      "reward": -1.1127638339996337,
+      "reward_std": 1.0112764954566955,
+      "step": 230,
+      "step_time": 379.6049673743546
+    },
+    {
+      "clip_ratio/high_max": 0.0013269021990709006,
+      "clip_ratio/high_mean": 0.000772348974714987,
+      "clip_ratio/low_mean": 0.006830424629151821,
+      "clip_ratio/low_min": 0.001417644842331356,
+      "clip_ratio/region_mean": 0.007602773635881022,
+      "completion_length": 7148.85234375,
+      "completions/clipped_ratio": 0.96484375,
+      "completions/max_length": 15185.6,
+      "completions/max_terminated_length": 10879.4,
+      "completions/mean_length": 7148.85234375,
+      "completions/mean_terminated_length": 5810.6314453125,
+      "completions/min_length": 554.6,
+      "completions/min_terminated_length": 726.2,
+      "epoch": 0.7692307692307693,
+      "frac_reward_zero_std": 0.053125,
+      "grad_norm": 0.567713737487793,
+      "kl": 1.9632584005594254,
+      "learning_rate": 1e-06,
+      "loss": 0.0331,
+      "num_tokens": 187710108.0,
+      "reward": -1.6358975887298584,
+      "reward_std": 0.8105679512023926,
+      "step": 240,
+      "step_time": 364.4212720930576
+    },
+    {
+      "clip_ratio/high_max": 0.009734998072963208,
+      "clip_ratio/high_mean": 0.0036562698369380087,
+      "clip_ratio/low_mean": 0.0031687672017142177,
+      "clip_ratio/low_min": 0.0011266621499089525,
+      "clip_ratio/region_mean": 0.006825037160888314,
+      "completion_length": 7711.33046875,
+      "completions/clipped_ratio": 0.9703125,
+      "completions/max_length": 15326.6,
+      "completions/max_terminated_length": 8454.8,
+      "completions/mean_length": 7711.33046875,
+      "completions/mean_terminated_length": 4505.596484375,
+      "completions/min_length": 507.4,
+      "completions/min_terminated_length": 627.8,
+      "epoch": 0.8012820512820513,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.301693856716156,
+      "kl": 2.305810421705246,
+      "learning_rate": 1e-06,
+      "loss": 0.0411,
+      "num_tokens": 198946847.0,
+      "reward": -1.7136600971221925,
+      "reward_std": 0.7779677748680115,
+      "step": 250,
+      "step_time": 366.3410928696394
+    },
+    {
+      "clip_ratio/high_max": 0.005677101551555097,
+      "clip_ratio/high_mean": 0.0026270149246556684,
+      "clip_ratio/low_mean": 0.007492635349626653,
+      "clip_ratio/low_min": 0.002146819365589181,
+      "clip_ratio/region_mean": 0.01011965008219704,
+      "completion_length": 7928.271875,
+      "completions/clipped_ratio": 0.97578125,
+      "completions/max_length": 15486.4,
+      "completions/max_terminated_length": 8957.6,
+      "completions/mean_length": 7928.271875,
+      "completions/mean_terminated_length": 4968.226708984375,
+      "completions/min_length": 330.6,
+      "completions/min_terminated_length": 810.6,
+      "epoch": 0.8333333333333334,
+      "frac_reward_zero_std": 0.04375,
+      "grad_norm": 0.1900310516357422,
+      "kl": 3.3469204664230348,
+      "learning_rate": 1e-06,
+      "loss": 0.058,
+      "num_tokens": 210451663.0,
+      "reward": -1.5978996515274049,
+      "reward_std": 0.8638608932495118,
+      "step": 260,
+      "step_time": 369.1461088169366
+    },
+    {
+      "clip_ratio/high_max": 0.0053559043677523736,
+      "clip_ratio/high_mean": 0.0023730095708742738,
+      "clip_ratio/low_mean": 0.003168451989768073,
+      "clip_ratio/low_min": 0.0007724208910076414,
+      "clip_ratio/region_mean": 0.0055414614733308555,
+      "completion_length": 7882.1359375,
+      "completions/clipped_ratio": 0.9359375,
+      "completions/max_length": 15262.0,
+      "completions/max_terminated_length": 10161.4,
+      "completions/mean_length": 7882.1359375,
+      "completions/mean_terminated_length": 4356.723779296875,
+      "completions/min_length": 478.2,
+      "completions/min_terminated_length": 771.2,
+      "epoch": 0.8653846153846154,
+      "frac_reward_zero_std": 0.015625,
+      "grad_norm": 0.2651360034942627,
+      "kl": 4.203655362129211,
+      "learning_rate": 1e-06,
+      "loss": 0.0717,
+      "num_tokens": 221911233.0,
+      "reward": -1.3130165576934814,
+      "reward_std": 0.9633238554000855,
+      "step": 270,
+      "step_time": 376.3476559590548
+    },
+    {
+      "clip_ratio/high_max": 0.004202661872841418,
+      "clip_ratio/high_mean": 0.001565266537363641,
+      "clip_ratio/low_mean": 0.005889438977465034,
+      "clip_ratio/low_min": 0.0018291531931026838,
+      "clip_ratio/region_mean": 0.0074547056807205084,
+      "completion_length": 8012.12109375,
+      "completions/clipped_ratio": 0.9265625,
+      "completions/max_length": 15195.8,
+      "completions/max_terminated_length": 11226.8,
+      "completions/mean_length": 8012.12109375,
+      "completions/mean_terminated_length": 4799.8126953125,
+      "completions/min_length": 376.4,
+      "completions/min_terminated_length": 461.2,
+      "epoch": 0.8974358974358975,
+      "frac_reward_zero_std": 0.021875,
+      "grad_norm": 0.8448399305343628,
+      "kl": 6.057727479934693,
+      "learning_rate": 1e-06,
+      "loss": 0.0929,
+      "num_tokens": 233560444.0,
+      "reward": -0.9053005456924439,
+      "reward_std": 1.0727339029312133,
+      "step": 280,
+      "step_time": 366.69956831745804
+    },
+    {
+      "clip_ratio/high_max": 0.009579948079772294,
+      "clip_ratio/high_mean": 0.003866587582160719,
+      "clip_ratio/low_mean": 0.026560859876917677,
+      "clip_ratio/low_min": 0.005657196008542087,
+      "clip_ratio/region_mean": 0.03042744711274281,
+      "completion_length": 8332.26171875,
+      "completions/clipped_ratio": 0.9484375,
+      "completions/max_length": 15238.0,
+      "completions/max_terminated_length": 9476.0,
+      "completions/mean_length": 8332.26171875,
+      "completions/mean_terminated_length": 4191.986083984375,
+      "completions/min_length": 314.2,
+      "completions/min_terminated_length": 459.0,
+      "epoch": 0.9294871794871795,
+      "frac_reward_zero_std": 0.00625,
+      "grad_norm": 0.7789781093597412,
+      "kl": 7.377506709098816,
+      "learning_rate": 1e-06,
+      "loss": 0.1126,
+      "num_tokens": 245602039.0,
+      "reward": -0.9568447232246399,
+      "reward_std": 1.1172706007957458,
+      "step": 290,
+      "step_time": 369.8988025188446
+    },
+    {
+      "clip_ratio/high_max": 0.04103642666013911,
+      "clip_ratio/high_mean": 0.015878588767373004,
+      "clip_ratio/low_mean": 0.002557483545388095,
+      "clip_ratio/low_min": 0.0008715285985090304,
+      "clip_ratio/region_mean": 0.018436072464101016,
+      "completion_length": 8640.715625,
+      "completions/clipped_ratio": 0.95546875,
+      "completions/max_length": 15339.4,
+      "completions/max_terminated_length": 9081.8,
+      "completions/mean_length": 8640.715625,
+      "completions/mean_terminated_length": 3938.00859375,
+      "completions/min_length": 401.0,
+      "completions/min_terminated_length": 461.2,
+      "epoch": 0.9615384615384616,
+      "frac_reward_zero_std": 0.015625,
+      "grad_norm": 1.6522005796432495,
+      "kl": 8.443660879135132,
+      "learning_rate": 1e-06,
+      "loss": 0.1394,
+      "num_tokens": 257971567.0,
+      "reward": -0.9969817280769349,
+      "reward_std": 1.1187695264816284,
+      "step": 300,
+      "step_time": 380.1039554864168
+    },
+    {
+      "clip_ratio/high_max": 0.019450648140627892,
+      "clip_ratio/high_mean": 0.006475560626131483,
+      "clip_ratio/low_mean": 0.00424157283268869,
+      "clip_ratio/low_min": 0.0011720009730197488,
+      "clip_ratio/region_mean": 0.010717133258003742,
+      "completion_length": 8589.09296875,
+      "completions/clipped_ratio": 0.9671875,
+      "completions/max_length": 15468.2,
+      "completions/max_terminated_length": 7561.2,
+      "completions/mean_length": 8589.09296875,
+      "completions/mean_terminated_length": 3331.67998046875,
+      "completions/min_length": 322.8,
+      "completions/min_terminated_length": 332.6,
+      "epoch": 0.9935897435897436,
+      "frac_reward_zero_std": 0.03125,
+      "grad_norm": 0.5063915848731995,
+      "kl": 8.70184623003006,
+      "learning_rate": 1e-06,
+      "loss": 0.1401,
+      "num_tokens": 270328654.0,
+      "reward": -1.1191815495491029,
+      "reward_std": 1.1197254419326783,
+      "step": 310,
+      "step_time": 368.0659532535821
+    },
+    {
+      "clip_ratio/high_max": 0.008593197559821419,
+      "clip_ratio/high_mean": 0.0025123804312897845,
+      "clip_ratio/low_mean": 0.004413481615483761,
+      "clip_ratio/low_min": 0.0007776515500154346,
+      "clip_ratio/region_mean": 0.0069258620031178,
+      "completion_length": 8683.9328125,
+      "completions/clipped_ratio": 0.96796875,
+      "completions/max_length": 15360.4,
+      "completions/max_terminated_length": 6737.0,
+      "completions/mean_length": 8683.9328125,
+      "completions/mean_terminated_length": 2616.1105712890626,
+      "completions/min_length": 301.0,
+      "completions/min_terminated_length": 317.0,
+      "epoch": 1.0256410256410255,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.6627429127693176,
+      "kl": 8.537155032157898,
+      "learning_rate": 1e-06,
+      "loss": 0.1324,
+      "num_tokens": 282829496.0,
+      "reward": -1.1347467184066773,
+      "reward_std": 1.1445826292037964,
+      "step": 320,
+      "step_time": 387.8881539706141
+    },
+    {
+      "clip_ratio/high_max": 0.014052026980789379,
+      "clip_ratio/high_mean": 0.005751634493935853,
+      "clip_ratio/low_mean": 0.03248874342534691,
+      "clip_ratio/low_min": 0.01050121024177315,
+      "clip_ratio/region_mean": 0.03824037743615918,
+      "completion_length": 8227.85625,
+      "completions/clipped_ratio": 0.9640625,
+      "completions/max_length": 15314.0,
+      "completions/max_terminated_length": 7328.4,
+      "completions/mean_length": 8227.85625,
+      "completions/mean_terminated_length": 1938.225244140625,
+      "completions/min_length": 369.6,
+      "completions/min_terminated_length": 372.8,
+      "epoch": 1.0576923076923077,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.6000649333000183,
+      "kl": 8.414502596855163,
+      "learning_rate": 1e-06,
+      "loss": 0.1263,
+      "num_tokens": 294739692.0,
+      "reward": -0.9467703104019165,
+      "reward_std": 1.1953829765319823,
+      "step": 330,
+      "step_time": 381.7925847005099
+    },
+    {
+      "clip_ratio/high_max": 0.02248516906693112,
+      "clip_ratio/high_mean": 0.006262405202141963,
+      "clip_ratio/low_mean": 0.0028700845839921387,
+      "clip_ratio/low_min": 0.0009011315094539895,
+      "clip_ratio/region_mean": 0.009132489818148315,
+      "completion_length": 8437.27890625,
+      "completions/clipped_ratio": 0.9625,
+      "completions/max_length": 15303.2,
+      "completions/max_terminated_length": 4607.2,
+      "completions/mean_length": 8437.27890625,
+      "completions/mean_terminated_length": 1254.5227661132812,
+      "completions/min_length": 236.8,
+      "completions/min_terminated_length": 276.4,
+      "epoch": 1.0897435897435896,
+      "frac_reward_zero_std": 0.015625,
+      "grad_norm": 0.31412163376808167,
+      "kl": 9.649739027023315,
+      "learning_rate": 1e-06,
+      "loss": 0.1534,
+      "num_tokens": 306936745.0,
+      "reward": -1.0691025137901307,
+      "reward_std": 1.156248641014099,
+      "step": 340,
+      "step_time": 379.1928739707917
+    },
+    {
+      "clip_ratio/high_max": 0.001976413730881177,
+      "clip_ratio/high_mean": 0.001007653510896489,
+      "clip_ratio/low_mean": 0.002886987855890766,
+      "clip_ratio/low_min": 0.0006830676909885369,
+      "clip_ratio/region_mean": 0.0038946413900703193,
+      "completion_length": 8375.2421875,
+      "completions/clipped_ratio": 0.971875,
+      "completions/max_length": 15308.0,
+      "completions/max_terminated_length": 7826.2,
+      "completions/mean_length": 8375.2421875,
+      "completions/mean_terminated_length": 2755.6521728515627,
+      "completions/min_length": 431.2,
+      "completions/min_terminated_length": 438.6,
+      "epoch": 1.1217948717948718,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.3878633379936218,
+      "kl": 9.245597887039185,
+      "learning_rate": 1e-06,
+      "loss": 0.1483,
+      "num_tokens": 318971499.0,
+      "reward": -1.0583446741104126,
+      "reward_std": 1.2148149728775024,
+      "step": 350,
+      "step_time": 379.80885707810523
+    },
+    {
+      "clip_ratio/high_max": 0.001718334387987852,
+      "clip_ratio/high_mean": 0.0009115864173509181,
+      "clip_ratio/low_mean": 0.005361310031730682,
+      "clip_ratio/low_min": 0.0009540307913994184,
+      "clip_ratio/region_mean": 0.006272896577138453,
+      "completion_length": 8509.92109375,
+      "completions/clipped_ratio": 0.97578125,
+      "completions/max_length": 15296.0,
+      "completions/max_terminated_length": 6474.0,
+      "completions/mean_length": 8509.92109375,
+      "completions/mean_terminated_length": 2695.9841064453126,
+      "completions/min_length": 559.0,
+      "completions/min_terminated_length": 955.4,
+      "epoch": 1.1538461538461537,
+      "frac_reward_zero_std": 0.003125,
+      "grad_norm": 1.318275809288025,
+      "kl": 9.4658158659935,
+      "learning_rate": 1e-06,
+      "loss": 0.1449,
+      "num_tokens": 331218394.0,
+      "reward": -0.9336093187332153,
+      "reward_std": 1.2436002016067504,
+      "step": 360,
+      "step_time": 385.18268774822354
+    },
+    {
+      "clip_ratio/high_max": 0.02834329553297721,
+      "clip_ratio/high_mean": 0.011119978004717267,
+      "clip_ratio/low_mean": 0.0037480033381143583,
+      "clip_ratio/low_min": 0.0009545553322823253,
+      "clip_ratio/region_mean": 0.014867981395218522,
+      "completion_length": 8731.73828125,
+      "completions/clipped_ratio": 0.97421875,
+      "completions/max_length": 15342.6,
+      "completions/max_terminated_length": 4419.8,
+      "completions/mean_length": 8731.73828125,
+      "completions/mean_terminated_length": 2164.4700317382812,
+      "completions/min_length": 575.8,
+      "completions/min_terminated_length": 606.0,
+      "epoch": 1.185897435897436,
+      "frac_reward_zero_std": 0.009375,
+      "grad_norm": 0.9024655222892761,
+      "kl": 9.558997631072998,
+      "learning_rate": 1e-06,
+      "loss": 0.1544,
+      "num_tokens": 343791735.0,
+      "reward": -1.0594681978225708,
+      "reward_std": 1.1597766399383544,
+      "step": 370,
+      "step_time": 400.0213452059776
+    },
+    {
+      "clip_ratio/high_max": 0.017531970868003555,
+      "clip_ratio/high_mean": 0.007362127504893579,
+      "clip_ratio/low_mean": 0.012022353924112394,
+      "clip_ratio/low_min": 0.002931717267347267,
+      "clip_ratio/region_mean": 0.019384481851011513,
+      "completion_length": 8895.28125,
+      "completions/clipped_ratio": 0.96875,
+      "completions/max_length": 15392.6,
+      "completions/max_terminated_length": 1901.8,
+      "completions/mean_length": 8895.28125,
+      "completions/mean_terminated_length": 1199.2500244140624,
+      "completions/min_length": 316.6,
+      "completions/min_terminated_length": 593.0,
+      "epoch": 1.217948717948718,
+      "frac_reward_zero_std": 0.00625,
+      "grad_norm": 0.7320740818977356,
+      "kl": 8.970777094364166,
+      "learning_rate": 1e-06,
+      "loss": 0.1445,
+      "num_tokens": 356565719.0,
+      "reward": -1.129196548461914,
+      "reward_std": 1.1558172225952148,
+      "step": 380,
+      "step_time": 397.49473525583744
+    },
+    {
+      "clip_ratio/high_max": 0.0019310896430397406,
+      "clip_ratio/high_mean": 0.0010317706852219998,
+      "clip_ratio/low_mean": 0.007496347883716226,
+      "clip_ratio/low_min": 0.0013817044891766272,
+      "clip_ratio/region_mean": 0.008528118708636612,
+      "completion_length": 8823.5828125,
+      "completions/clipped_ratio": 0.97734375,
+      "completions/max_length": 15380.0,
+      "completions/max_terminated_length": 2765.0,
+      "completions/mean_length": 8823.5828125,
+      "completions/mean_terminated_length": 1701.2597778320312,
+      "completions/min_length": 373.6,
+      "completions/min_terminated_length": 637.2,
+      "epoch": 1.25,
+      "frac_reward_zero_std": 0.015625,
+      "grad_norm": 0.44049158692359924,
+      "kl": 8.034697198867798,
+      "learning_rate": 1e-06,
+      "loss": 0.1267,
+      "num_tokens": 369274933.0,
+      "reward": -1.1584037780761718,
+      "reward_std": 1.18006112575531,
+      "step": 390,
+      "step_time": 382.658811436221
+    },
+    {
+      "clip_ratio/high_max": 0.009981066177715547,
+      "clip_ratio/high_mean": 0.0038592111959587783,
+      "clip_ratio/low_mean": 0.008657075540395454,
+      "clip_ratio/low_min": 0.002548298077454092,
+      "clip_ratio/region_mean": 0.012516286806203425,
+      "completion_length": 8711.2421875,
+      "completions/clipped_ratio": 0.96484375,
+      "completions/max_length": 15438.4,
+      "completions/max_terminated_length": 3635.8,
+      "completions/mean_length": 8711.2421875,
+      "completions/mean_terminated_length": 1193.8060546875,
+      "completions/min_length": 375.4,
+      "completions/min_terminated_length": 398.2,
+      "epoch": 1.282051282051282,
+      "frac_reward_zero_std": 0.003125,
+      "grad_norm": 0.9623905420303345,
+      "kl": 9.493924081325531,
+      "learning_rate": 1e-06,
+      "loss": 0.1494,
+      "num_tokens": 381813531.0,
+      "reward": -1.066390573978424,
+      "reward_std": 1.22670156955719,
+      "step": 400,
+      "step_time": 392.0767021100968
+    },
+    {
+      "clip_ratio/high_max": 0.00988367916434072,
+      "clip_ratio/high_mean": 0.00355387544259429,
+      "clip_ratio/low_mean": 0.00904853937390726,
+      "clip_ratio/low_min": 0.0011502065903187032,
+      "clip_ratio/region_mean": 0.012602414819411933,
+      "completion_length": 8496.709375,
+      "completions/clipped_ratio": 0.97421875,
+      "completions/max_length": 15298.2,
+      "completions/max_terminated_length": 2271.8,
+      "completions/mean_length": 8496.709375,
+      "completions/mean_terminated_length": 841.8000244140625,
+      "completions/min_length": 237.4,
+      "completions/min_terminated_length": 237.4,
+      "epoch": 1.314102564102564,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.1496130228042603,
+      "kl": 8.819581186771392,
+      "learning_rate": 1e-06,
+      "loss": 0.1379,
+      "num_tokens": 394018999.0,
+      "reward": -1.0360162615776063,
+      "reward_std": 1.230124545097351,
+      "step": 410,
+      "step_time": 391.23668179214
+    },
+    {
+      "clip_ratio/high_max": 0.026965406071394683,
+      "clip_ratio/high_mean": 0.008239620376843958,
+      "clip_ratio/low_mean": 0.005393524651299231,
+      "clip_ratio/low_min": 0.001032962081808364,
+      "clip_ratio/region_mean": 0.013633145205676555,
+      "completion_length": 8715.7203125,
+      "completions/clipped_ratio": 0.9671875,
+      "completions/max_length": 15293.4,
+      "completions/max_terminated_length": 6496.2,
+      "completions/mean_length": 8715.7203125,
+      "completions/mean_terminated_length": 1550.4044677734375,
+      "completions/min_length": 272.4,
+      "completions/min_terminated_length": 272.4,
+      "epoch": 1.3461538461538463,
+      "frac_reward_zero_std": 0.009375,
+      "grad_norm": 1.0314459800720215,
+      "kl": 8.277751541137695,
+      "learning_rate": 1e-06,
+      "loss": 0.1413,
+      "num_tokens": 406564613.0,
+      "reward": -1.1630311250686645,
+      "reward_std": 1.1583751916885376,
+      "step": 420,
+      "step_time": 403.144832425192
+    },
+    {
+      "clip_ratio/high_max": 0.02055753368185833,
+      "clip_ratio/high_mean": 0.007283784722676501,
+      "clip_ratio/low_mean": 0.006037392208236269,
+      "clip_ratio/low_min": 0.0013361667824938195,
+      "clip_ratio/region_mean": 0.013321176473982631,
+      "completion_length": 9150.809375,
+      "completions/clipped_ratio": 0.97421875,
+      "completions/max_length": 15317.6,
+      "completions/max_terminated_length": 4995.6,
+      "completions/mean_length": 9150.809375,
+      "completions/mean_terminated_length": 1580.1365539550782,
+      "completions/min_length": 421.4,
+      "completions/min_terminated_length": 421.4,
+      "epoch": 1.3782051282051282,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 1.3688066005706787,
+      "kl": 8.256977772712707,
+      "learning_rate": 1e-06,
+      "loss": 0.1447,
+      "num_tokens": 419656069.0,
+      "reward": -1.3635391473770142,
+      "reward_std": 1.0858495473861693,
+      "step": 430,
+      "step_time": 381.2247460927814
+    },
+    {
+      "clip_ratio/high_max": 0.03355933935381472,
+      "clip_ratio/high_mean": 0.010686016583349556,
+      "clip_ratio/low_mean": 0.00426324947620742,
+      "clip_ratio/low_min": 0.0006483774355729111,
+      "clip_ratio/region_mean": 0.01494926589075476,
+      "completion_length": 9334.15546875,
+      "completions/clipped_ratio": 0.95859375,
+      "completions/max_length": 15303.0,
+      "completions/max_terminated_length": 1801.0,
+      "completions/mean_length": 9334.15546875,
+      "completions/mean_terminated_length": 748.5221557617188,
+      "completions/min_length": 189.6,
+      "completions/min_terminated_length": 189.6,
+      "epoch": 1.4102564102564101,
+      "frac_reward_zero_std": 0.00625,
+      "grad_norm": 0.6061592102050781,
+      "kl": 9.276415371894837,
+      "learning_rate": 1e-06,
+      "loss": 0.1649,
+      "num_tokens": 432945296.0,
+      "reward": -1.6430940866470336,
+      "reward_std": 0.8856900691986084,
+      "step": 440,
+      "step_time": 397.19712854139505
+    },
+    {
+      "clip_ratio/high_max": 0.025904236652422696,
+      "clip_ratio/high_mean": 0.010497866358491592,
+      "clip_ratio/low_mean": 0.009116584471485112,
+      "clip_ratio/low_min": 0.0006169638683786616,
+      "clip_ratio/region_mean": 0.01961445074994117,
+      "completion_length": 9772.859375,
+      "completions/clipped_ratio": 0.97734375,
+      "completions/max_length": 15328.4,
+      "completions/max_terminated_length": 2742.2,
+      "completions/mean_length": 9772.859375,
+      "completions/mean_terminated_length": 1179.7389038085937,
+      "completions/min_length": 269.6,
+      "completions/min_terminated_length": 269.6,
+      "epoch": 1.4423076923076923,
+      "frac_reward_zero_std": 0.00625,
+      "grad_norm": 2.9273993968963623,
+      "kl": 5.355518877506256,
+      "learning_rate": 1e-06,
+      "loss": 0.1,
+      "num_tokens": 446795924.0,
+      "reward": -1.7697789430618287,
+      "reward_std": 0.7878645062446594,
+      "step": 450,
+      "step_time": 393.0521178711206
+    },
+    {
+      "clip_ratio/high_max": 0.03093774657463655,
+      "clip_ratio/high_mean": 0.010312027207692153,
+      "clip_ratio/low_mean": 0.018413475481793284,
+      "clip_ratio/low_min": 0.004762036592001096,
+      "clip_ratio/region_mean": 0.028725503571331502,
+      "completion_length": 9846.35078125,
+      "completions/clipped_ratio": 0.96953125,
+      "completions/max_length": 15342.8,
+      "completions/max_terminated_length": 1341.4,
+      "completions/mean_length": 9846.35078125,
+      "completions/mean_terminated_length": 843.5957336425781,
+      "completions/min_length": 416.4,
+      "completions/min_terminated_length": 482.4,
+      "epoch": 1.4743589743589745,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.9283716678619385,
+      "kl": 6.169983702898025,
+      "learning_rate": 1e-06,
+      "loss": 0.1131,
+      "num_tokens": 460783829.0,
+      "reward": -1.8763149499893188,
+      "reward_std": 0.7387312293052674,
+      "step": 460,
+      "step_time": 380.59655289798974
+    },
+    {
+      "clip_ratio/high_max": 0.030389591585844755,
+      "clip_ratio/high_mean": 0.010781910654623061,
+      "clip_ratio/low_mean": 0.0014117073966190218,
+      "clip_ratio/low_min": 0.0004272733713150956,
+      "clip_ratio/region_mean": 0.012193618039600552,
+      "completion_length": 9743.57578125,
+      "completions/clipped_ratio": 0.978125,
+      "completions/max_length": 15434.4,
+      "completions/max_terminated_length": 1303.4,
+      "completions/mean_length": 9743.57578125,
+      "completions/mean_terminated_length": 568.9239013671875,
+      "completions/min_length": 458.6,
+      "completions/min_terminated_length": 233.0,
+      "epoch": 1.5064102564102564,
+      "frac_reward_zero_std": 0.00625,
+      "grad_norm": 1.246910572052002,
+      "kl": 5.609658390283585,
+      "learning_rate": 1e-06,
+      "loss": 0.1033,
+      "num_tokens": 474602646.0,
+      "reward": -1.5660094499588013,
+      "reward_std": 0.9299165487289429,
+      "step": 470,
+      "step_time": 364.2963201098144
+    },
+    {
+      "clip_ratio/high_max": 0.04028984218603,
+      "clip_ratio/high_mean": 0.010068922629579901,
+      "clip_ratio/low_mean": 0.005153270141454414,
+      "clip_ratio/low_min": 0.0010639606771292165,
+      "clip_ratio/region_mean": 0.015222192776855081,
+      "completion_length": 10232.215625,
+      "completions/clipped_ratio": 0.97578125,
+      "completions/max_length": 15349.4,
+      "completions/max_terminated_length": 965.4,
+      "completions/mean_length": 10232.215625,
+      "completions/mean_terminated_length": 575.2595397949219,
+      "completions/min_length": 274.0,
+      "completions/min_terminated_length": 274.0,
+      "epoch": 1.5384615384615383,
+      "frac_reward_zero_std": 0.003125,
+      "grad_norm": 5.255829811096191,
+      "kl": 6.466679322719574,
+      "learning_rate": 1e-06,
+      "loss": 0.128,
+      "num_tokens": 489084942.0,
+      "reward": -1.7843213319778441,
+      "reward_std": 0.7446226239204407,
+      "step": 480,
+      "step_time": 386.39945622831584
+    },
+    {
+      "clip_ratio/high_max": 0.021522707428084688,
+      "clip_ratio/high_mean": 0.005565525087877177,
+      "clip_ratio/low_mean": 0.005207951000193134,
+      "clip_ratio/low_min": 0.000600995840068208,
+      "clip_ratio/region_mean": 0.010773476003669203,
+      "completion_length": 10032.759375,
+      "completions/clipped_ratio": 0.9765625,
+      "completions/max_length": 15285.6,
+      "completions/max_terminated_length": 855.2,
+      "completions/mean_length": 10032.759375,
+      "completions/mean_terminated_length": 559.0505126953125,
+      "completions/min_length": 255.8,
+      "completions/min_terminated_length": 301.2,
+      "epoch": 1.5705128205128205,
+      "frac_reward_zero_std": 0.003125,
+      "grad_norm": 1.1097595691680908,
+      "kl": 5.85886058807373,
+      "learning_rate": 1e-06,
+      "loss": 0.11,
+      "num_tokens": 503289954.0,
+      "reward": -1.6432590723037719,
+      "reward_std": 0.8262911438941956,
+      "step": 490,
+      "step_time": 382.7899776160717
+    },
+    {
+      "clip_ratio/high_max": 0.01541918225120753,
+      "clip_ratio/high_mean": 0.004005147295538336,
+      "clip_ratio/low_mean": 0.011318963055964559,
+      "clip_ratio/low_min": 0.0007145337311158073,
+      "clip_ratio/region_mean": 0.015324109909124672,
+      "completion_length": 10167.6828125,
+      "completions/clipped_ratio": 0.97578125,
+      "completions/max_length": 15193.2,
+      "completions/max_terminated_length": 1639.0,
+      "completions/mean_length": 10167.6828125,
+      "completions/mean_terminated_length": 1071.7352661132813,
+      "completions/min_length": 589.2,
+      "completions/min_terminated_length": 653.6,
+      "epoch": 1.6025641025641026,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 2.3733479976654053,
+      "kl": 6.069680732488632,
+      "learning_rate": 1e-06,
+      "loss": 0.1188,
+      "num_tokens": 517686000.0,
+      "reward": -1.5672321796417237,
+      "reward_std": 0.8680948257446289,
+      "step": 500,
+      "step_time": 397.2997827142477
     }
   ],
   "logging_steps": 10,
-  "max_steps": 78,
-  "num_input_tokens_seen": 49637699,
-  "num_train_epochs": 1,
-  "save_steps": 16,
+  "max_steps": 624,
+  "num_input_tokens_seen": 517686000,
+  "num_train_epochs": 2,
+  "save_steps": 125,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {
@@ -156,7 +1328,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 16,
+  "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:97f58e63a5761f9f87a4c01a6bb9f6a12db1eb6eeb08944f741a561f393d8e3b
-size 9041
+oid sha256:f0a9201c977a939be903256ae14044a58c07828bdf03798edbf3bfae6e4a2e58
+size 9169