Model save

README.md

@@ -27,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/ryanzhang123-the-university-of-auckland/RLCR/runs/sh7jv7qj) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/ryanzhang123-the-university-of-auckland/RLCR/runs/an9do2om) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
@@ -35,10 +35,10 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 ### Framework versions
 
 - TRL: 0.16.0.dev0
-- Transformers: 4.48.3
+- Transformers: 4.57.6
 - Pytorch: 2.5.1
 - Datasets: 4.0.0
-- Tokenizers: 0.21.4
+- Tokenizers: 0.22.2
 
 ## Citations
 

chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

config.json

@@ -1,28 +1,57 @@
 {
-  "_name_or_path": "Qwen/Qwen2.5-1.5B-Instruct",
   "architectures": [
     "Qwen2ForCausalLM"
   ],
   "attention_dropout": 0.0,
-  "bos_token_id": 151643,
+  "dtype": "bfloat16",
   "eos_token_id": 151645,
   "hidden_act": "silu",
   "hidden_size": 1536,
   "initializer_range": 0.02,
   "intermediate_size": 8960,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
   "max_position_embeddings": 32768,
   "max_window_layers": 21,
   "model_type": "qwen2",
   "num_attention_heads": 12,
   "num_hidden_layers": 28,
   "num_key_value_heads": 2,
+  "pad_token_id": 151643,
   "rms_norm_eps": 1e-06,
   "rope_scaling": null,
   "rope_theta": 1000000.0,
   "sliding_window": null,
   "tie_word_embeddings": true,
-  "torch_dtype": "bfloat16",
-  "transformers_version": "4.48.3",
+  "transformers_version": "4.57.6",
   "use_cache": false,
   "use_sliding_window": false,
   "vocab_size": 151936

generation_config.json

@@ -1,5 +1,4 @@
 {
-  "bos_token_id": 151643,
   "do_sample": true,
   "eos_token_id": [
     151645,
@@ -10,5 +9,5 @@
   "temperature": 0.7,
   "top_k": 20,
   "top_p": 0.8,
-  "transformers_version": "4.48.3"
+  "transformers_version": "4.57.6"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:847e81aee77c06a8f17ef0b268c4404b99e8adc993e47a7f4f24b280e4479f00
+oid sha256:7d1478beef101c30f89a0299831d5c5ad516e28ab13ce2f54cac15c86e320f02
 size 3554214752

tokenizer_config.json

@@ -195,7 +195,6 @@
     "<|video_pad|>"
   ],
   "bos_token": null,
-  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
   "errors": "replace",

trainer_state.json

@@ -1,9 +1,10 @@
 {
+  "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 2.9790660225442833,
-  "eval_steps": 50,
-  "global_step": 57,
+  "epoch": 1.0,
+  "eval_steps": 40,
+  "global_step": 69,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -14,24 +15,24 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0109375,
-      "completions/max_length": 1536.0,
-      "completions/max_terminated_length": 1367.6,
-      "completions/mean_length": 234.5890625,
-      "completions/mean_terminated_length": 220.1892883300781,
-      "completions/min_length": 20.0,
-      "completions/min_terminated_length": 20.0,
-      "epoch": 0.2576489533011272,
-      "grad_norm": 0.0038548149168491364,
+      "completions/clipped_ratio": 0.003125,
+      "completions/max_length": 1221.6,
+      "completions/max_terminated_length": 1174.2,
+      "completions/mean_length": 256.43125,
+      "completions/mean_terminated_length": 252.51890258789064,
+      "completions/min_length": 46.0,
+      "completions/min_terminated_length": 46.0,
+      "epoch": 0.07246376811594203,
+      "grad_norm": 0.3151230216026306,
       "learning_rate": 1e-06,
-      "loss": 0.0398,
-      "num_tokens": 2520908.0,
-      "reward": 0.4615234375,
-      "reward_std": 0.22349842488765717,
-      "rewards/accuracy_reward": 0.091015625,
+      "loss": 0.0002,
+      "num_tokens": 328870.0,
+      "reward": 0.5421875,
+      "reward_std": 0.12776496410369872,
+      "rewards/accuracy_reward": 0.11875,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.83203125,
+      "rewards/format_reward": 0.965625,
       "rewards/mean_confidence_reward": 0.0,
       "step": 5
     },
@@ -41,24 +42,24 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.008984375,
-      "completions/max_length": 1536.0,
-      "completions/max_terminated_length": 1362.2,
-      "completions/mean_length": 203.15546875,
-      "completions/mean_terminated_length": 191.06714477539063,
-      "completions/min_length": 22.2,
-      "completions/min_terminated_length": 22.2,
-      "epoch": 0.5152979066022544,
-      "grad_norm": 0.002689502201974392,
+      "completions/clipped_ratio": 0.021875,
+      "completions/max_length": 1485.2,
+      "completions/max_terminated_length": 1166.8,
+      "completions/mean_length": 311.05625,
+      "completions/mean_terminated_length": 283.60601806640625,
+      "completions/min_length": 29.0,
+      "completions/min_terminated_length": 29.0,
+      "epoch": 0.14492753623188406,
+      "grad_norm": 0.17299839854240417,
       "learning_rate": 1e-06,
-      "loss": 0.0413,
-      "num_tokens": 4984682.0,
-      "reward": 0.5259765625,
-      "reward_std": 0.19930846989154816,
-      "rewards/accuracy_reward": 0.136328125,
+      "loss": 0.0051,
+      "num_tokens": 679604.0,
+      "reward": 0.5078125,
+      "reward_std": 0.13135676234960555,
+      "rewards/accuracy_reward": 0.090625,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.915625,
+      "rewards/format_reward": 0.925,
       "rewards/mean_confidence_reward": 0.0,
       "step": 10
     },
@@ -68,24 +69,24 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.006640625,
-      "completions/max_length": 1536.0,
-      "completions/max_terminated_length": 1360.2,
-      "completions/mean_length": 172.904296875,
-      "completions/mean_terminated_length": 163.77848205566406,
-      "completions/min_length": 22.4,
-      "completions/min_terminated_length": 22.4,
-      "epoch": 0.7729468599033816,
-      "grad_norm": 0.002530910074710846,
+      "completions/clipped_ratio": 0.009375,
+      "completions/max_length": 1389.2,
+      "completions/max_terminated_length": 1218.4,
+      "completions/mean_length": 271.359375,
+      "completions/mean_terminated_length": 259.41539001464844,
+      "completions/min_length": 35.2,
+      "completions/min_terminated_length": 35.2,
+      "epoch": 0.21739130434782608,
+      "grad_norm": 0.43196597695350647,
       "learning_rate": 1e-06,
-      "loss": 0.0315,
-      "num_tokens": 7340045.0,
-      "reward": 0.5544921875,
-      "reward_std": 0.16046438217163086,
-      "rewards/accuracy_reward": 0.14375,
+      "loss": 0.0196,
+      "num_tokens": 1028099.0,
+      "reward": 0.5421875,
+      "reward_std": 0.12912921607494354,
+      "rewards/accuracy_reward": 0.115625,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.965234375,
+      "rewards/format_reward": 0.96875,
       "rewards/mean_confidence_reward": 0.0,
       "step": 15
     },
@@ -95,24 +96,24 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0015625,
-      "completions/max_length": 1363.4,
-      "completions/max_terminated_length": 1173.4,
-      "completions/mean_length": 133.865380859375,
-      "completions/mean_terminated_length": 131.67113342285157,
-      "completions/min_length": 22.6,
-      "completions/min_terminated_length": 22.6,
-      "epoch": 1.0515297906602254,
-      "grad_norm": 0.0030923241283744574,
+      "completions/clipped_ratio": 0.00625,
+      "completions/max_length": 1289.6,
+      "completions/max_terminated_length": 1147.8,
+      "completions/mean_length": 252.15625,
+      "completions/mean_terminated_length": 244.12163391113282,
+      "completions/min_length": 36.2,
+      "completions/min_terminated_length": 36.2,
+      "epoch": 0.2898550724637681,
+      "grad_norm": 0.2922244966030121,
       "learning_rate": 1e-06,
-      "loss": 0.0202,
-      "num_tokens": 9588252.0,
-      "reward": 0.58359375,
-      "reward_std": 0.1627782851457596,
-      "rewards/accuracy_reward": 0.18359375,
+      "loss": -0.0002,
+      "num_tokens": 1365581.0,
+      "reward": 0.5578125,
+      "reward_std": 0.12591025978326797,
+      "rewards/accuracy_reward": 0.140625,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.98359375,
+      "rewards/format_reward": 0.975,
       "rewards/mean_confidence_reward": 0.0,
       "step": 20
     },
@@ -122,24 +123,24 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.00546875,
-      "completions/max_length": 1536.0,
-      "completions/max_terminated_length": 1325.6,
-      "completions/mean_length": 125.908984375,
-      "completions/mean_terminated_length": 118.147509765625,
-      "completions/min_length": 19.2,
-      "completions/min_terminated_length": 19.2,
-      "epoch": 1.3091787439613527,
-      "grad_norm": 0.003670709440484643,
+      "completions/clipped_ratio": 0.009375,
+      "completions/max_length": 1396.2,
+      "completions/max_terminated_length": 1287.2,
+      "completions/mean_length": 273.45625,
+      "completions/mean_terminated_length": 261.5423217773438,
+      "completions/min_length": 28.2,
+      "completions/min_terminated_length": 28.2,
+      "epoch": 0.36231884057971014,
+      "grad_norm": 0.4919992983341217,
       "learning_rate": 1e-06,
-      "loss": 0.0216,
-      "num_tokens": 11824499.0,
-      "reward": 0.5904296875,
-      "reward_std": 0.15798357725143433,
-      "rewards/accuracy_reward": 0.190234375,
+      "loss": 0.0035,
+      "num_tokens": 1705483.0,
+      "reward": 0.58125,
+      "reward_std": 0.14668796956539154,
+      "rewards/accuracy_reward": 0.184375,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.990625,
+      "rewards/format_reward": 0.978125,
       "rewards/mean_confidence_reward": 0.0,
       "step": 25
     },
@@ -149,24 +150,24 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.00625,
-      "completions/max_length": 1536.0,
-      "completions/max_terminated_length": 1178.6,
-      "completions/mean_length": 110.7203125,
-      "completions/mean_terminated_length": 101.76295471191406,
-      "completions/min_length": 19.6,
-      "completions/min_terminated_length": 19.6,
-      "epoch": 1.5668276972624797,
-      "grad_norm": 0.0037478143349289894,
+      "completions/clipped_ratio": 0.009375,
+      "completions/max_length": 1195.0,
+      "completions/max_terminated_length": 987.2,
+      "completions/mean_length": 171.409375,
+      "completions/mean_terminated_length": 159.10531311035157,
+      "completions/min_length": 13.8,
+      "completions/min_terminated_length": 13.8,
+      "epoch": 0.43478260869565216,
+      "grad_norm": 1.3107653856277466,
       "learning_rate": 1e-06,
-      "loss": 0.0245,
-      "num_tokens": 14029327.0,
-      "reward": 0.6072265625,
-      "reward_std": 0.18271726369857788,
-      "rewards/accuracy_reward": 0.229296875,
+      "loss": 0.0299,
+      "num_tokens": 2006542.0,
+      "reward": 0.6109375,
+      "reward_std": 0.1539909154176712,
+      "rewards/accuracy_reward": 0.24375,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.98515625,
+      "rewards/format_reward": 0.978125,
       "rewards/mean_confidence_reward": 0.0,
       "step": 30
     },
@@ -176,24 +177,24 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.00625,
-      "completions/max_length": 1536.0,
-      "completions/max_terminated_length": 1204.2,
-      "completions/mean_length": 102.811328125,
-      "completions/mean_terminated_length": 93.77486419677734,
-      "completions/min_length": 20.0,
-      "completions/min_terminated_length": 20.0,
-      "epoch": 1.8244766505636072,
-      "grad_norm": 0.004457506816834211,
+      "completions/clipped_ratio": 0.003125,
+      "completions/max_length": 545.2,
+      "completions/max_terminated_length": 472.8,
+      "completions/mean_length": 37.778125,
+      "completions/mean_terminated_length": 33.125397491455075,
+      "completions/min_length": 13.6,
+      "completions/min_terminated_length": 13.6,
+      "epoch": 0.5072463768115942,
+      "grad_norm": 0.5966821312904358,
       "learning_rate": 1e-06,
-      "loss": 0.028,
-      "num_tokens": 16205916.0,
-      "reward": 0.6328125,
-      "reward_std": 0.1903953105211258,
-      "rewards/accuracy_reward": 0.275390625,
+      "loss": 0.0217,
+      "num_tokens": 2276899.0,
+      "reward": 0.6671875,
+      "reward_std": 0.18611472547054292,
+      "rewards/accuracy_reward": 0.35625,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.990234375,
+      "rewards/format_reward": 0.978125,
       "rewards/mean_confidence_reward": 0.0,
       "step": 35
     },
@@ -203,51 +204,74 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.002734375,
-      "completions/max_length": 1379.6,
-      "completions/max_terminated_length": 943.4,
-      "completions/mean_length": 83.36419067382812,
-      "completions/mean_terminated_length": 79.37923889160156,
-      "completions/min_length": 19.2,
-      "completions/min_terminated_length": 19.2,
-      "epoch": 2.103059581320451,
-      "grad_norm": 0.004812587983906269,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 23.2,
+      "completions/max_terminated_length": 23.2,
+      "completions/mean_length": 14.140625,
+      "completions/mean_terminated_length": 14.140625,
+      "completions/min_length": 13.2,
+      "completions/min_terminated_length": 13.2,
+      "epoch": 0.5797101449275363,
+      "grad_norm": 0.3963298201560974,
       "learning_rate": 1e-06,
-      "loss": 0.0179,
-      "num_tokens": 18335851.0,
-      "reward": 0.6435546875,
-      "reward_std": 0.19647349417209625,
-      "rewards/accuracy_reward": 0.294140625,
+      "loss": 0.0008,
+      "num_tokens": 2537316.0,
+      "reward": 0.6640625,
+      "reward_std": 0.17168534100055693,
+      "rewards/accuracy_reward": 0.328125,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.99296875,
+      "rewards/format_reward": 1.0,
       "rewards/mean_confidence_reward": 0.0,
       "step": 40
     },
+    {
+      "epoch": 0.5797101449275363,
+      "eval_completions/clipped_ratio": 0.0009057971014492754,
+      "eval_completions/max_length": 32.666666666666664,
+      "eval_completions/max_terminated_length": 21.63768115942029,
+      "eval_completions/mean_length": 16.29981884057971,
+      "eval_completions/mean_terminated_length": 14.921066256536953,
+      "eval_completions/min_length": 13.73913043478261,
+      "eval_completions/min_terminated_length": 13.73913043478261,
+      "eval_loss": 0.0,
+      "eval_num_tokens": 2537316.0,
+      "eval_reward": 0.6947463768115942,
+      "eval_reward_std": 0.18031474883141724,
+      "eval_rewards/accuracy_reward": 0.39221014492753625,
+      "eval_rewards/brier_reward": 0.0,
+      "eval_rewards/confidence_one_or_zero": 0.0,
+      "eval_rewards/format_reward": 0.9972826086956522,
+      "eval_rewards/mean_confidence_reward": 0.0,
+      "eval_runtime": 40.463,
+      "eval_samples_per_second": 6.821,
+      "eval_steps_per_second": 0.865,
+      "step": 40
+    },
     {
       "clip_ratio/high_max": 0.0,
       "clip_ratio/high_mean": 0.0,
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.001171875,
-      "completions/max_length": 1318.2,
-      "completions/max_terminated_length": 1053.4,
-      "completions/mean_length": 76.5015625,
-      "completions/mean_terminated_length": 74.79784393310547,
-      "completions/min_length": 19.8,
-      "completions/min_terminated_length": 19.8,
-      "epoch": 2.360708534621578,
-      "grad_norm": 0.005865577142685652,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 14.6,
+      "completions/max_terminated_length": 14.6,
+      "completions/mean_length": 13.821875,
+      "completions/mean_terminated_length": 13.821875,
+      "completions/min_length": 13.0,
+      "completions/min_terminated_length": 13.0,
+      "epoch": 0.6521739130434783,
+      "grad_norm": 0.7279768586158752,
       "learning_rate": 1e-06,
-      "loss": 0.0019,
-      "num_tokens": 20453815.0,
-      "reward": 0.6810546875,
-      "reward_std": 0.20889041423797608,
-      "rewards/accuracy_reward": 0.366015625,
+      "loss": 0.0,
+      "num_tokens": 2786831.0,
+      "reward": 0.7421875,
+      "reward_std": 0.13392626643180847,
+      "rewards/accuracy_reward": 0.484375,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.99609375,
+      "rewards/format_reward": 1.0,
       "rewards/mean_confidence_reward": 0.0,
       "step": 45
     },
@@ -257,49 +281,53 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.002734375,
-      "completions/max_length": 1272.4,
-      "completions/max_terminated_length": 916.0,
-      "completions/mean_length": 69.940625,
-      "completions/mean_terminated_length": 65.93077697753907,
-      "completions/min_length": 19.2,
-      "completions/min_terminated_length": 19.2,
-      "epoch": 2.6183574879227054,
-      "grad_norm": 0.0054559651762247086,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 21.2,
+      "completions/max_terminated_length": 21.2,
+      "completions/mean_length": 13.4,
+      "completions/mean_terminated_length": 13.4,
+      "completions/min_length": 12.0,
+      "completions/min_terminated_length": 12.0,
+      "epoch": 0.7246376811594203,
+      "grad_norm": 0.6197125315666199,
       "learning_rate": 1e-06,
-      "loss": 0.0238,
-      "num_tokens": 22524927.0,
-      "reward": 0.6978515625,
-      "reward_std": 0.20110094547271729,
-      "rewards/accuracy_reward": 0.40078125,
+      "loss": 0.0008,
+      "num_tokens": 3037315.0,
+      "reward": 0.7390625,
+      "reward_std": 0.07284187749028206,
+      "rewards/accuracy_reward": 0.48125,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.994921875,
+      "rewards/format_reward": 0.996875,
       "rewards/mean_confidence_reward": 0.0,
       "step": 50
     },
     {
-      "epoch": 2.6183574879227054,
-      "eval_completions/clipped_ratio": 0.0018115942028985507,
-      "eval_completions/max_length": 216.3913043478261,
-      "eval_completions/max_terminated_length": 194.92753623188406,
-      "eval_completions/mean_length": 67.3913043478261,
-      "eval_completions/mean_terminated_length": 64.67624224787173,
-      "eval_completions/min_length": 27.32608695652174,
-      "eval_completions/min_terminated_length": 27.32608695652174,
-      "eval_loss": 0.0,
-      "eval_num_tokens": 22524927.0,
-      "eval_reward": 0.7137681159420289,
-      "eval_reward_std": 0.1941313454206439,
-      "eval_rewards/accuracy_reward": 0.4311594202898551,
-      "eval_rewards/brier_reward": 0.0,
-      "eval_rewards/confidence_one_or_zero": 0.0,
-      "eval_rewards/format_reward": 0.9963768115942029,
-      "eval_rewards/mean_confidence_reward": 0.0,
-      "eval_runtime": 167.5343,
-      "eval_samples_per_second": 0.824,
-      "eval_steps_per_second": 0.107,
-      "step": 50
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 14.0,
+      "completions/max_terminated_length": 14.0,
+      "completions/mean_length": 13.290625,
+      "completions/mean_terminated_length": 13.290625,
+      "completions/min_length": 13.0,
+      "completions/min_terminated_length": 13.0,
+      "epoch": 0.7971014492753623,
+      "grad_norm": 0.0,
+      "learning_rate": 1e-06,
+      "loss": -0.0,
+      "num_tokens": 3284236.0,
+      "reward": 0.76875,
+      "reward_std": 0.03221687823534012,
+      "rewards/accuracy_reward": 0.5375,
+      "rewards/brier_reward": 0.0,
+      "rewards/confidence_one_or_zero": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/mean_confidence_reward": 0.0,
+      "step": 55
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -307,26 +335,53 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.003125,
-      "completions/max_length": 1536.0,
-      "completions/max_terminated_length": 955.6,
-      "completions/mean_length": 68.77109375,
-      "completions/mean_terminated_length": 64.17131195068359,
-      "completions/min_length": 20.0,
-      "completions/min_terminated_length": 20.0,
-      "epoch": 2.8760064412238324,
-      "grad_norm": 0.23794075846672058,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 14.0,
+      "completions/max_terminated_length": 14.0,
+      "completions/mean_length": 13.21875,
+      "completions/mean_terminated_length": 13.21875,
+      "completions/min_length": 13.0,
+      "completions/min_terminated_length": 13.0,
+      "epoch": 0.8695652173913043,
+      "grad_norm": 0.0,
       "learning_rate": 1e-06,
-      "loss": 0.0176,
-      "num_tokens": 24644237.0,
-      "reward": 0.7162109375,
-      "reward_std": 0.17682999968528748,
-      "rewards/accuracy_reward": 0.43671875,
+      "loss": 0.0001,
+      "num_tokens": 3538302.0,
+      "reward": 0.7109375,
+      "reward_std": 0.003125,
+      "rewards/accuracy_reward": 0.421875,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.995703125,
+      "rewards/format_reward": 1.0,
       "rewards/mean_confidence_reward": 0.0,
-      "step": 55
+      "step": 60
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 13.4,
+      "completions/max_terminated_length": 13.4,
+      "completions/mean_length": 13.015625,
+      "completions/mean_terminated_length": 13.015625,
+      "completions/min_length": 13.0,
+      "completions/min_terminated_length": 13.0,
+      "epoch": 0.9420289855072463,
+      "grad_norm": 0.0,
+      "learning_rate": 1e-06,
+      "loss": 0.0,
+      "num_tokens": 3798055.0,
+      "reward": 0.7375,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.475,
+      "rewards/brier_reward": 0.0,
+      "rewards/confidence_one_or_zero": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/mean_confidence_reward": 0.0,
+      "step": 65
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -334,35 +389,35 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0022935779816513624,
-      "completions/max_length": 1237.5,
-      "completions/max_terminated_length": 820.5,
-      "completions/mean_length": 64.20463943481445,
-      "completions/mean_terminated_length": 60.82253074645996,
-      "completions/min_length": 18.5,
-      "completions/min_terminated_length": 18.5,
-      "epoch": 2.9790660225442833,
-      "num_tokens": 25486243.0,
-      "reward": 0.71484375,
-      "reward_std": 0.15983812510967255,
-      "rewards/accuracy_reward": 0.43359375,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 13.0,
+      "completions/max_terminated_length": 13.0,
+      "completions/mean_length": 13.0,
+      "completions/mean_terminated_length": 13.0,
+      "completions/min_length": 13.0,
+      "completions/min_terminated_length": 13.0,
+      "epoch": 1.0,
+      "num_tokens": 4000983.0,
+      "reward": 0.763671875,
+      "reward_std": 0.00390625,
+      "rewards/accuracy_reward": 0.52734375,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.99609375,
+      "rewards/format_reward": 1.0,
       "rewards/mean_confidence_reward": 0.0,
-      "step": 57,
+      "step": 69,
       "total_flos": 0.0,
-      "train_loss": 0.024099274620152357,
-      "train_runtime": 6308.3505,
-      "train_samples_per_second": 0.591,
-      "train_steps_per_second": 0.009
+      "train_loss": 0.005885362614998999,
+      "train_runtime": 1112.4293,
+      "train_samples_per_second": 0.992,
+      "train_steps_per_second": 0.062
     }
   ],
   "logging_steps": 5,
-  "max_steps": 57,
-  "num_input_tokens_seen": 25486243,
-  "num_train_epochs": 3,
-  "save_steps": 60,
+  "max_steps": 69,
+  "num_input_tokens_seen": 4000983,
+  "num_train_epochs": 1,
+  "save_steps": 40,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {
@@ -376,7 +431,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 8,
+  "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:56fac7b61fc3b10834b75eafc732a9f5bc8c79e2030753b838bff9789c8b803d
-size 8056
+oid sha256:d36c815be02d35da2ef1110a70e716729d767538210445941f93bed0f42a44c2
+size 8120